From 5fdd3b39bb8150d1ea8622e42e0166ed46af7693 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Fri, 12 Apr 2024 23:22:20 +0900 Subject: [PATCH 01/37] Allow 1 count block request to return 0 blocks (#5554) * Allow 1 count block request to return 0 blocks * Address @pawanjay176 review --- .../lighthouse_network/src/rpc/handler.rs | 81 +++++++++---------- .../lighthouse_network/src/rpc/methods.rs | 21 ----- .../lighthouse_network/src/rpc/outbound.rs | 17 +++- .../lighthouse_network/src/rpc/protocol.rs | 4 +- .../src/rpc/rate_limiter.rs | 12 +-- 5 files changed, 63 insertions(+), 72 deletions(-) diff --git a/beacon_node/lighthouse_network/src/rpc/handler.rs b/beacon_node/lighthouse_network/src/rpc/handler.rs index cb537f23590..df5bbba99c8 100644 --- a/beacon_node/lighthouse_network/src/rpc/handler.rs +++ b/beacon_node/lighthouse_network/src/rpc/handler.rs @@ -163,7 +163,7 @@ struct InboundInfo { /// Protocol of the original request we received from the peer. protocol: Protocol, /// Responses that the peer is still expecting from us. - remaining_chunks: u64, + max_remaining_chunks: u64, /// Useful to timing how long each request took to process. Currently only used by /// BlocksByRange. request_start_time: Instant, @@ -180,7 +180,7 @@ struct OutboundInfo { /// Info over the protocol this substream is handling. proto: Protocol, /// Number of chunks to be seen from the peer's response. - remaining_chunks: Option, + max_remaining_chunks: Option, /// `Id` as given by the application that sent the request. req_id: Id, } @@ -471,7 +471,7 @@ where // Process one more message if one exists. if let Some(message) = info.pending_items.pop_front() { // If this is the last chunk, terminate the stream. - let last_chunk = info.remaining_chunks <= 1; + let last_chunk = info.max_remaining_chunks <= 1; let fut = send_message_to_inbound_substream(substream, message, last_chunk) .boxed(); @@ -537,7 +537,8 @@ where { // The substream is still active, decrement the remaining // chunks expected. - info.remaining_chunks = info.remaining_chunks.saturating_sub(1); + info.max_remaining_chunks = + info.max_remaining_chunks.saturating_sub(1); // If this substream has not ended, we reset the timer. // Each chunk is allowed RESPONSE_TIMEOUT to be sent. @@ -552,7 +553,7 @@ where // Process one more message if one exists. if let Some(message) = info.pending_items.pop_front() { // If this is the last chunk, terminate the stream. - let last_chunk = info.remaining_chunks <= 1; + let last_chunk = info.max_remaining_chunks <= 1; let fut = send_message_to_inbound_substream( substream, message, last_chunk, ) @@ -664,15 +665,19 @@ where request, } => match substream.poll_next_unpin(cx) { Poll::Ready(Some(Ok(response))) => { - if request.expected_responses() > 1 && !response.close_after() { + if request.expect_exactly_one_response() || response.close_after() { + // either this is a single response request or this response closes the + // stream + entry.get_mut().state = OutboundSubstreamState::Closing(substream); + } else { let substream_entry = entry.get_mut(); let delay_key = &substream_entry.delay_key; // chunks left after this one - let remaining_chunks = substream_entry - .remaining_chunks + let max_remaining_chunks = substream_entry + .max_remaining_chunks .map(|count| count.saturating_sub(1)) .unwrap_or_else(|| 0); - if remaining_chunks == 0 { + if max_remaining_chunks == 0 { // this is the last expected message, close the stream as all expected chunks have been received substream_entry.state = OutboundSubstreamState::Closing(substream); } else { @@ -682,14 +687,10 @@ where substream, request, }; - substream_entry.remaining_chunks = Some(remaining_chunks); + substream_entry.max_remaining_chunks = Some(max_remaining_chunks); self.outbound_substreams_delay .reset(delay_key, self.resp_timeout); } - } else { - // either this is a single response request or this response closes the - // stream - entry.get_mut().state = OutboundSubstreamState::Closing(substream); } // Check what type of response we got and report it accordingly @@ -725,7 +726,16 @@ where self.outbound_substreams_delay.remove(delay_key); entry.remove_entry(); // notify the application error - if request.expected_responses() > 1 { + if request.expect_exactly_one_response() { + // return an error, stream should not have closed early. + return Poll::Ready(ConnectionHandlerEvent::NotifyBehaviour( + HandlerEvent::Err(HandlerErr::Outbound { + id: request_id, + proto: request.versioned_protocol().protocol(), + error: RPCError::IncompleteStream, + }), + )); + } else { // return an end of stream result return Poll::Ready(ConnectionHandlerEvent::NotifyBehaviour( HandlerEvent::Ok(RPCReceived::EndOfStream( @@ -734,16 +744,6 @@ where )), )); } - - // else we return an error, stream should not have closed early. - let outbound_err = HandlerErr::Outbound { - id: request_id, - proto: request.versioned_protocol().protocol(), - error: RPCError::IncompleteStream, - }; - return Poll::Ready(ConnectionHandlerEvent::NotifyBehaviour( - HandlerEvent::Err(outbound_err), - )); } Poll::Pending => { entry.get_mut().state = @@ -880,10 +880,10 @@ where } let (req, substream) = substream; - let expected_responses = req.expected_responses(); + let max_responses = req.max_responses(); // store requests that expect responses - if expected_responses > 0 { + if max_responses > 0 { if self.inbound_substreams.len() < MAX_INBOUND_SUBSTREAMS { // Store the stream and tag the output. let delay_key = self @@ -894,14 +894,13 @@ where self.current_inbound_substream_id, InboundInfo { state: awaiting_stream, - pending_items: VecDeque::with_capacity(std::cmp::min( - expected_responses, - 128, - ) as usize), + pending_items: VecDeque::with_capacity( + std::cmp::min(max_responses, 128) as usize + ), delay_key: Some(delay_key), protocol: req.versioned_protocol().protocol(), request_start_time: Instant::now(), - remaining_chunks: expected_responses, + max_remaining_chunks: max_responses, }, ); } else { @@ -948,8 +947,14 @@ where } // add the stream to substreams if we expect a response, otherwise drop the stream. - let expected_responses = request.expected_responses(); - if expected_responses > 0 { + let max_responses = request.max_responses(); + if max_responses > 0 { + let max_remaining_chunks = if request.expect_exactly_one_response() { + // Currently enforced only for multiple responses + None + } else { + Some(max_responses) + }; // new outbound request. Store the stream and tag the output. let delay_key = self .outbound_substreams_delay @@ -958,12 +963,6 @@ where substream: Box::new(substream), request, }; - let expected_responses = if expected_responses > 1 { - // Currently enforced only for multiple responses - Some(expected_responses) - } else { - None - }; if self .outbound_substreams .insert( @@ -972,7 +971,7 @@ where state: awaiting_stream, delay_key, proto, - remaining_chunks: expected_responses, + max_remaining_chunks, req_id: id, }, ) diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs index 67eea09ea77..1b0486ff771 100644 --- a/beacon_node/lighthouse_network/src/rpc/methods.rs +++ b/beacon_node/lighthouse_network/src/rpc/methods.rs @@ -483,27 +483,6 @@ impl RPCCodedResponse { RPCCodedResponse::Error(code, err) } - /// Specifies which response allows for multiple chunks for the stream handler. - pub fn multiple_responses(&self) -> bool { - match self { - RPCCodedResponse::Success(resp) => match resp { - RPCResponse::Status(_) => false, - RPCResponse::BlocksByRange(_) => true, - RPCResponse::BlocksByRoot(_) => true, - RPCResponse::BlobsByRange(_) => true, - RPCResponse::BlobsByRoot(_) => true, - RPCResponse::Pong(_) => false, - RPCResponse::MetaData(_) => false, - RPCResponse::LightClientBootstrap(_) => false, - RPCResponse::LightClientOptimisticUpdate(_) => false, - RPCResponse::LightClientFinalityUpdate(_) => false, - }, - RPCCodedResponse::Error(_, _) => true, - // Stream terminations are part of responses that have chunks - RPCCodedResponse::StreamTermination(_) => true, - } - } - /// Returns true if this response always terminates the stream. pub fn close_after(&self) -> bool { !matches!(self, RPCCodedResponse::Success(_)) diff --git a/beacon_node/lighthouse_network/src/rpc/outbound.rs b/beacon_node/lighthouse_network/src/rpc/outbound.rs index 09ef6185957..8ea7b84bc95 100644 --- a/beacon_node/lighthouse_network/src/rpc/outbound.rs +++ b/beacon_node/lighthouse_network/src/rpc/outbound.rs @@ -91,8 +91,8 @@ impl OutboundRequest { } /* These functions are used in the handler for stream management */ - /// Number of responses expected for this request. - pub fn expected_responses(&self) -> u64 { + /// Maximum number of responses expected for this request. + pub fn max_responses(&self) -> u64 { match self { OutboundRequest::Status(_) => 1, OutboundRequest::Goodbye(_) => 0, @@ -105,6 +105,19 @@ impl OutboundRequest { } } + pub fn expect_exactly_one_response(&self) -> bool { + match self { + OutboundRequest::Status(_) => true, + OutboundRequest::Goodbye(_) => false, + OutboundRequest::BlocksByRange(_) => false, + OutboundRequest::BlocksByRoot(_) => false, + OutboundRequest::BlobsByRange(_) => false, + OutboundRequest::BlobsByRoot(_) => false, + OutboundRequest::Ping(_) => true, + OutboundRequest::MetaData(_) => true, + } + } + /// Gives the corresponding `SupportedProtocol` to this request. pub fn versioned_protocol(&self) -> SupportedProtocol { match self { diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index 6ff72f658a8..f65586087c2 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -654,8 +654,8 @@ pub enum InboundRequest { impl InboundRequest { /* These functions are used in the handler for stream management */ - /// Number of responses expected for this request. - pub fn expected_responses(&self) -> u64 { + /// Maximum number of responses expected for this request. + pub fn max_responses(&self) -> u64 { match self { InboundRequest::Status(_) => 1, InboundRequest::Goodbye(_) => 0, diff --git a/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs b/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs index 801a4af54b3..b304eb546da 100644 --- a/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs +++ b/beacon_node/lighthouse_network/src/rpc/rate_limiter.rs @@ -228,7 +228,7 @@ impl RPCRateLimiterBuilder { pub trait RateLimiterItem { fn protocol(&self) -> Protocol; - fn expected_responses(&self) -> u64; + fn max_responses(&self) -> u64; } impl RateLimiterItem for super::InboundRequest { @@ -236,8 +236,8 @@ impl RateLimiterItem for super::InboundRequest { self.versioned_protocol().protocol() } - fn expected_responses(&self) -> u64 { - self.expected_responses() + fn max_responses(&self) -> u64 { + self.max_responses() } } @@ -246,8 +246,8 @@ impl RateLimiterItem for super::OutboundRequest { self.versioned_protocol().protocol() } - fn expected_responses(&self) -> u64 { - self.expected_responses() + fn max_responses(&self) -> u64 { + self.max_responses() } } impl RPCRateLimiter { @@ -299,7 +299,7 @@ impl RPCRateLimiter { request: &Item, ) -> Result<(), RateLimitedErr> { let time_since_start = self.init_time.elapsed(); - let tokens = request.expected_responses().max(1); + let tokens = request.max_responses().max(1); let check = |limiter: &mut Limiter| limiter.allows(time_since_start, peer_id, tokens); From 6fb0b2ed788926676a1e1c943714825868e1546f Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Sat, 13 Apr 2024 00:39:11 +0900 Subject: [PATCH 02/37] Sync lookup dedup range and blobs (#5561) * Handle sync range blocks as blocks and blobs * Merge range sync and backfill sync handling * Update tests * Add no_blobs_into_responses test * Address @realbigsean comments * Merge remote-tracking branch 'origin/unstable' into sync-lookup-dedup-range-and-blobs --- beacon_node/network/src/router.rs | 15 +- .../network/src/sync/backfill_sync/mod.rs | 8 +- .../src/sync/block_sidecar_coupling.rs | 54 ++- beacon_node/network/src/sync/manager.rs | 259 +++---------- .../network/src/sync/network_context.rs | 343 ++++-------------- .../network/src/sync/range_sync/chain.rs | 11 +- .../network/src/sync/range_sync/range.rs | 97 ++--- 7 files changed, 256 insertions(+), 531 deletions(-) diff --git a/beacon_node/network/src/router.rs b/beacon_node/network/src/router.rs index 2f7a92e91ed..1937fc11cf9 100644 --- a/beacon_node/network/src/router.rs +++ b/beacon_node/network/src/router.rs @@ -497,10 +497,7 @@ impl Router { crit!(self.log, "Block lookups do not request BBRange requests"; "peer_id" => %peer_id); return; } - id @ (SyncId::BackFillBlocks { .. } - | SyncId::RangeBlocks { .. } - | SyncId::BackFillBlockAndBlobs { .. } - | SyncId::RangeBlockAndBlobs { .. }) => id, + id @ SyncId::RangeBlockAndBlobs { .. } => id, }, RequestId::Router => { crit!(self.log, "All BBRange requests belong to sync"; "peer_id" => %peer_id); @@ -559,10 +556,7 @@ impl Router { let request_id = match request_id { RequestId::Sync(sync_id) => match sync_id { id @ SyncId::SingleBlock { .. } => id, - SyncId::BackFillBlocks { .. } - | SyncId::RangeBlocks { .. } - | SyncId::RangeBlockAndBlobs { .. } - | SyncId::BackFillBlockAndBlobs { .. } => { + SyncId::RangeBlockAndBlobs { .. } => { crit!(self.log, "Batch syncing do not request BBRoot requests"; "peer_id" => %peer_id); return; } @@ -604,10 +598,7 @@ impl Router { crit!(self.log, "Block response to blobs by roots request"; "peer_id" => %peer_id); return; } - SyncId::BackFillBlocks { .. } - | SyncId::RangeBlocks { .. } - | SyncId::RangeBlockAndBlobs { .. } - | SyncId::BackFillBlockAndBlobs { .. } => { + SyncId::RangeBlockAndBlobs { .. } => { crit!(self.log, "Batch syncing does not request BBRoot requests"; "peer_id" => %peer_id); return; } diff --git a/beacon_node/network/src/sync/backfill_sync/mod.rs b/beacon_node/network/src/sync/backfill_sync/mod.rs index 915aeb82ec4..67fe871accf 100644 --- a/beacon_node/network/src/sync/backfill_sync/mod.rs +++ b/beacon_node/network/src/sync/backfill_sync/mod.rs @@ -10,6 +10,7 @@ use crate::network_beacon_processor::ChainSegmentProcessId; use crate::sync::manager::{BatchProcessResult, Id}; +use crate::sync::network_context::RangeRequestId; use crate::sync::network_context::SyncNetworkContext; use crate::sync::range_sync::{ BatchConfig, BatchId, BatchInfo, BatchOperationOutcome, BatchProcessingResult, BatchState, @@ -961,7 +962,12 @@ impl BackFillSync { ) -> Result<(), BackFillError> { if let Some(batch) = self.batches.get_mut(&batch_id) { let (request, is_blob_batch) = batch.to_blocks_by_range_request(); - match network.backfill_blocks_by_range_request(peer, is_blob_batch, request, batch_id) { + match network.blocks_and_blobs_by_range_request( + peer, + is_blob_batch, + request, + RangeRequestId::BackfillSync { batch_id }, + ) { Ok(request_id) => { // inform the batch about the new request if let Err(e) = batch.start_downloading_from_peer(peer, request_id) { diff --git a/beacon_node/network/src/sync/block_sidecar_coupling.rs b/beacon_node/network/src/sync/block_sidecar_coupling.rs index 9c6e2fcf07c..6a3b568c1c4 100644 --- a/beacon_node/network/src/sync/block_sidecar_coupling.rs +++ b/beacon_node/network/src/sync/block_sidecar_coupling.rs @@ -3,7 +3,9 @@ use ssz_types::VariableList; use std::{collections::VecDeque, sync::Arc}; use types::{BlobSidecar, EthSpec, SignedBeaconBlock}; -#[derive(Debug, Default)] +use super::range_sync::ByRangeRequestType; + +#[derive(Debug)] pub struct BlocksAndBlobsRequestInfo { /// Blocks we have received awaiting for their corresponding sidecar. accumulated_blocks: VecDeque>>, @@ -13,9 +15,25 @@ pub struct BlocksAndBlobsRequestInfo { is_blocks_stream_terminated: bool, /// Whether the individual RPC request for sidecars is finished or not. is_sidecars_stream_terminated: bool, + /// Used to determine if this accumulator should wait for a sidecars stream termination + request_type: ByRangeRequestType, } impl BlocksAndBlobsRequestInfo { + pub fn new(request_type: ByRangeRequestType) -> Self { + Self { + accumulated_blocks: <_>::default(), + accumulated_sidecars: <_>::default(), + is_blocks_stream_terminated: <_>::default(), + is_sidecars_stream_terminated: <_>::default(), + request_type, + } + } + + pub fn get_request_type(&self) -> ByRangeRequestType { + self.request_type + } + pub fn add_block_response(&mut self, block_opt: Option>>) { match block_opt { Some(block) => self.accumulated_blocks.push_back(block), @@ -78,6 +96,38 @@ impl BlocksAndBlobsRequestInfo { } pub fn is_finished(&self) -> bool { - self.is_blocks_stream_terminated && self.is_sidecars_stream_terminated + let blobs_requested = match self.request_type { + ByRangeRequestType::Blocks => false, + ByRangeRequestType::BlocksAndBlobs => true, + }; + self.is_blocks_stream_terminated && (!blobs_requested || self.is_sidecars_stream_terminated) + } +} + +#[cfg(test)] +mod tests { + use super::BlocksAndBlobsRequestInfo; + use crate::sync::range_sync::ByRangeRequestType; + use beacon_chain::test_utils::{generate_rand_block_and_blobs, NumBlobs}; + use rand::SeedableRng; + use types::{test_utils::XorShiftRng, ForkName, MinimalEthSpec as E}; + + #[test] + fn no_blobs_into_responses() { + let mut info = BlocksAndBlobsRequestInfo::::new(ByRangeRequestType::Blocks); + let mut rng = XorShiftRng::from_seed([42; 16]); + let blocks = (0..4) + .map(|_| generate_rand_block_and_blobs::(ForkName::Base, NumBlobs::None, &mut rng).0) + .collect::>(); + + // Send blocks and complete terminate response + for block in blocks { + info.add_block_response(Some(block.into())); + } + info.add_block_response(None); + + // Assert response is finished and RpcBlocks can be constructed + assert!(info.is_finished()); + info.into_responses().unwrap(); } } diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index ca4d0a2183d..15595ecde5c 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -36,7 +36,7 @@ use super::backfill_sync::{BackFillSync, ProcessResult, SyncStart}; use super::block_lookups::common::LookupType; use super::block_lookups::BlockLookups; -use super::network_context::{BlockOrBlob, SyncNetworkContext}; +use super::network_context::{BlockOrBlob, RangeRequestId, SyncNetworkContext}; use super::peer_sync_info::{remote_sync_type, PeerSyncType}; use super::range_sync::{RangeSync, RangeSyncType, EPOCHS_PER_BATCH}; use crate::network_beacon_processor::{ChainSegmentProcessId, NetworkBeaconProcessor}; @@ -44,8 +44,7 @@ use crate::service::NetworkMessage; use crate::status::ToStatusMessage; use crate::sync::block_lookups::common::{Current, Parent}; use crate::sync::block_lookups::{BlobRequestState, BlockRequestState}; -use crate::sync::network_context::BlocksAndBlobsByRangeRequest; -use crate::sync::range_sync::ByRangeRequestType; +use crate::sync::block_sidecar_coupling::BlocksAndBlobsRequestInfo; use beacon_chain::block_verification_types::AsBlock; use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::data_availability_checker::ChildComponents; @@ -91,12 +90,6 @@ pub enum RequestId { SingleBlock { id: SingleLookupReqId }, /// Request searching for a set of blobs given a hash. SingleBlob { id: SingleLookupReqId }, - /// Request was from the backfill sync algorithm. - BackFillBlocks { id: Id }, - /// Backfill request that is composed by both a block range request and a blob range request. - BackFillBlockAndBlobs { id: Id }, - /// The request was from a chain in the range sync algorithm. - RangeBlocks { id: Id }, /// Range request that is composed by both a block range request and a blob range request. RangeBlockAndBlobs { id: Id }, } @@ -363,63 +356,27 @@ impl SyncManager { error, ), }, - RequestId::BackFillBlocks { id } => { - if let Some(batch_id) = self - .network - .backfill_request_failed(id, ByRangeRequestType::Blocks) - { - match self - .backfill_sync - .inject_error(&mut self.network, batch_id, &peer_id, id) - { - Ok(_) => {} - Err(_) => self.update_sync_state(), - } - } - } - - RequestId::BackFillBlockAndBlobs { id } => { - if let Some(batch_id) = self - .network - .backfill_request_failed(id, ByRangeRequestType::BlocksAndBlobs) - { - match self - .backfill_sync - .inject_error(&mut self.network, batch_id, &peer_id, id) - { - Ok(_) => {} - Err(_) => self.update_sync_state(), - } - } - } - RequestId::RangeBlocks { id } => { - if let Some((chain_id, batch_id)) = self - .network - .range_sync_request_failed(id, ByRangeRequestType::Blocks) - { - self.range_sync.inject_error( - &mut self.network, - peer_id, - batch_id, - chain_id, - id, - ); - self.update_sync_state() - } - } RequestId::RangeBlockAndBlobs { id } => { - if let Some((chain_id, batch_id)) = self - .network - .range_sync_request_failed(id, ByRangeRequestType::BlocksAndBlobs) - { - self.range_sync.inject_error( - &mut self.network, - peer_id, - batch_id, - chain_id, - id, - ); - self.update_sync_state() + if let Some(sender_id) = self.network.range_request_failed(id) { + match sender_id { + RangeRequestId::RangeSync { chain_id, batch_id } => { + self.range_sync.inject_error( + &mut self.network, + peer_id, + batch_id, + chain_id, + id, + ); + self.update_sync_state(); + } + RangeRequestId::BackfillSync { batch_id } => match self + .backfill_sync + .inject_error(&mut self.network, batch_id, &peer_id, id) + { + Ok(_) => {} + Err(_) => self.update_sync_state(), + }, + } } } } @@ -901,49 +858,6 @@ impl SyncManager { RequestId::SingleBlob { .. } => { crit!(self.log, "Block received during blob request"; "peer_id" => %peer_id ); } - RequestId::BackFillBlocks { id } => { - let is_stream_terminator = block.is_none(); - if let Some(batch_id) = self - .network - .backfill_sync_only_blocks_response(id, is_stream_terminator) - { - match self.backfill_sync.on_block_response( - &mut self.network, - batch_id, - &peer_id, - id, - block.map(|b| RpcBlock::new_without_blobs(None, b)), - ) { - Ok(ProcessResult::SyncCompleted) => self.update_sync_state(), - Ok(ProcessResult::Successful) => {} - Err(_error) => { - // The backfill sync has failed, errors are reported - // within. - self.update_sync_state(); - } - } - } - } - RequestId::RangeBlocks { id } => { - let is_stream_terminator = block.is_none(); - if let Some((chain_id, batch_id)) = self - .network - .range_sync_block_only_response(id, is_stream_terminator) - { - self.range_sync.blocks_by_range_response( - &mut self.network, - peer_id, - chain_id, - batch_id, - id, - block.map(|b| RpcBlock::new_without_blobs(None, b)), - ); - self.update_sync_state(); - } - } - RequestId::BackFillBlockAndBlobs { id } => { - self.backfill_block_and_blobs_response(id, peer_id, block.into()) - } RequestId::RangeBlockAndBlobs { id } => { self.range_block_and_blobs_response(id, peer_id, block.into()) } @@ -981,15 +895,6 @@ impl SyncManager { &self.network, ), }, - RequestId::BackFillBlocks { id: _ } => { - crit!(self.log, "Blob received during backfill block request"; "peer_id" => %peer_id ); - } - RequestId::RangeBlocks { id: _ } => { - crit!(self.log, "Blob received during range block request"; "peer_id" => %peer_id ); - } - RequestId::BackFillBlockAndBlobs { id } => { - self.backfill_block_and_blobs_response(id, peer_id, blob.into()) - } RequestId::RangeBlockAndBlobs { id } => { self.range_block_and_blobs_response(id, peer_id, blob.into()) } @@ -1004,9 +909,9 @@ impl SyncManager { peer_id: PeerId, block_or_blob: BlockOrBlob, ) { - if let Some((chain_id, resp)) = self + if let Some(resp) = self .network - .range_sync_block_and_blob_response(id, block_or_blob) + .range_block_and_blob_response(id, block_or_blob) { match resp.responses { Ok(blocks) => { @@ -1016,33 +921,52 @@ impl SyncManager { // chain the stream terminator .chain(vec![None]) { - self.range_sync.blocks_by_range_response( - &mut self.network, - peer_id, - chain_id, - resp.batch_id, - id, - block, - ); - self.update_sync_state(); + match resp.sender_id { + RangeRequestId::RangeSync { chain_id, batch_id } => { + self.range_sync.blocks_by_range_response( + &mut self.network, + peer_id, + chain_id, + batch_id, + id, + block, + ); + self.update_sync_state(); + } + RangeRequestId::BackfillSync { batch_id } => { + match self.backfill_sync.on_block_response( + &mut self.network, + batch_id, + &peer_id, + id, + block, + ) { + Ok(ProcessResult::SyncCompleted) => self.update_sync_state(), + Ok(ProcessResult::Successful) => {} + Err(_error) => { + // The backfill sync has failed, errors are reported + // within. + self.update_sync_state(); + } + } + } + } } } Err(e) => { // Re-insert the request so we can retry - let new_req = BlocksAndBlobsByRangeRequest { - chain_id, - batch_id: resp.batch_id, - block_blob_info: <_>::default(), - }; - self.network - .insert_range_blocks_and_blobs_request(id, new_req); + self.network.insert_range_blocks_and_blobs_request( + id, + resp.sender_id, + BlocksAndBlobsRequestInfo::new(resp.request_type), + ); // inform range that the request needs to be treated as failed // With time we will want to downgrade this log warn!( self.log, "Blocks and blobs request for range received invalid data"; "peer_id" => %peer_id, - "batch_id" => resp.batch_id, + "sender_id" => ?resp.sender_id, "error" => e.clone() ); let id = RequestId::RangeBlockAndBlobs { id }; @@ -1056,69 +980,6 @@ impl SyncManager { } } } - - /// Handles receiving a response for a Backfill sync request that should have both blocks and - /// blobs. - fn backfill_block_and_blobs_response( - &mut self, - id: Id, - peer_id: PeerId, - block_or_blob: BlockOrBlob, - ) { - if let Some(resp) = self - .network - .backfill_sync_block_and_blob_response(id, block_or_blob) - { - match resp.responses { - Ok(blocks) => { - for block in blocks - .into_iter() - .map(Some) - // chain the stream terminator - .chain(vec![None]) - { - match self.backfill_sync.on_block_response( - &mut self.network, - resp.batch_id, - &peer_id, - id, - block, - ) { - Ok(ProcessResult::SyncCompleted) => self.update_sync_state(), - Ok(ProcessResult::Successful) => {} - Err(_error) => { - // The backfill sync has failed, errors are reported - // within. - self.update_sync_state(); - } - } - } - } - Err(e) => { - // Re-insert the request so we can retry - self.network.insert_backfill_blocks_and_blobs_requests( - id, - resp.batch_id, - <_>::default(), - ); - - // inform backfill that the request needs to be treated as failed - // With time we will want to downgrade this log - warn!( - self.log, "Blocks and blobs request for backfill received invalid data"; - "peer_id" => %peer_id, "batch_id" => resp.batch_id, "error" => e.clone() - ); - let id = RequestId::BackFillBlockAndBlobs { id }; - self.network.report_peer( - peer_id, - PeerAction::MidToleranceError, - "block_blob_faulty_backfill_batch", - ); - self.inject_error(peer_id, id, RPCError::InvalidData(e)) - } - } - } - } } impl From>> diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 29effb7bce0..96f8de46fb7 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -21,14 +21,20 @@ use tokio::sync::mpsc; use types::{BlobSidecar, EthSpec, SignedBeaconBlock}; pub struct BlocksAndBlobsByRangeResponse { - pub batch_id: BatchId, + pub sender_id: RangeRequestId, pub responses: Result>, String>, + pub request_type: ByRangeRequestType, } -pub struct BlocksAndBlobsByRangeRequest { - pub chain_id: ChainId, - pub batch_id: BatchId, - pub block_blob_info: BlocksAndBlobsRequestInfo, +#[derive(Debug, Clone, Copy)] +pub enum RangeRequestId { + RangeSync { + chain_id: ChainId, + batch_id: BatchId, + }, + BackfillSync { + batch_id: BatchId, + }, } /// Wraps a Network channel to employ various RPC related network functionality for the Sync manager. This includes management of a global RPC request Id. @@ -39,18 +45,9 @@ pub struct SyncNetworkContext { /// A sequential ID for all RPC requests. request_id: Id, - /// BlocksByRange requests made by the range syncing algorithm. - range_requests: FnvHashMap, - - /// BlocksByRange requests made by backfill syncing. - backfill_requests: FnvHashMap, - - /// BlocksByRange requests paired with BlobsByRange requests made by the range. - range_blocks_and_blobs_requests: FnvHashMap>, - - /// BlocksByRange requests paired with BlobsByRange requests made by the backfill sync. - backfill_blocks_and_blobs_requests: - FnvHashMap)>, + /// BlocksByRange requests paired with BlobsByRange + range_blocks_and_blobs_requests: + FnvHashMap)>, /// Whether the ee is online. If it's not, we don't allow access to the /// `beacon_processor_send`. @@ -94,10 +91,7 @@ impl SyncNetworkContext { network_send, execution_engine_state: EngineState::Online, // always assume `Online` at the start request_id: 1, - range_requests: FnvHashMap::default(), - backfill_requests: FnvHashMap::default(), range_blocks_and_blobs_requests: FnvHashMap::default(), - backfill_blocks_and_blobs_requests: FnvHashMap::default(), network_beacon_processor, chain, log, @@ -148,266 +142,85 @@ impl SyncNetworkContext { peer_id: PeerId, batch_type: ByRangeRequestType, request: BlocksByRangeRequest, - chain_id: ChainId, - batch_id: BatchId, ) -> Result { - match batch_type { - ByRangeRequestType::Blocks => { - trace!( - self.log, - "Sending BlocksByRange request"; - "method" => "BlocksByRange", - "count" => request.count(), - "peer" => %peer_id, - ); - let request = Request::BlocksByRange(request); - let id = self.next_id(); - let request_id = RequestId::Sync(SyncRequestId::RangeBlocks { id }); - self.send_network_msg(NetworkMessage::SendRequest { - peer_id, - request, - request_id, - })?; - self.range_requests.insert(id, (chain_id, batch_id)); - Ok(id) - } - ByRangeRequestType::BlocksAndBlobs => { - debug!( - self.log, - "Sending BlocksByRange and BlobsByRange requests"; - "method" => "Mixed by range request", - "count" => request.count(), - "peer" => %peer_id, - ); - - // create the shared request id. This is fine since the rpc handles substream ids. - let id = self.next_id(); - let request_id = RequestId::Sync(SyncRequestId::RangeBlockAndBlobs { id }); - - // Create the blob request based on the blob request. - let blobs_request = Request::BlobsByRange(BlobsByRangeRequest { + let id = self.next_id(); + trace!( + self.log, + "Sending BlocksByRange request"; + "method" => "BlocksByRange", + "count" => request.count(), + "peer" => %peer_id, + ); + self.send_network_msg(NetworkMessage::SendRequest { + peer_id, + request: Request::BlocksByRange(request.clone()), + request_id: RequestId::Sync(SyncRequestId::RangeBlockAndBlobs { id }), + })?; + + if matches!(batch_type, ByRangeRequestType::BlocksAndBlobs) { + debug!( + self.log, + "Sending BlobsByRange requests"; + "method" => "BlobsByRange", + "count" => request.count(), + "peer" => %peer_id, + ); + + // Create the blob request based on the blocks request. + self.send_network_msg(NetworkMessage::SendRequest { + peer_id, + request: Request::BlobsByRange(BlobsByRangeRequest { start_slot: *request.start_slot(), count: *request.count(), - }); - let blocks_request = Request::BlocksByRange(request); - - // Send both requests. Make sure both can be sent. - self.send_network_msg(NetworkMessage::SendRequest { - peer_id, - request: blocks_request, - request_id, - })?; - self.send_network_msg(NetworkMessage::SendRequest { - peer_id, - request: blobs_request, - request_id, - })?; - let block_blob_info = BlocksAndBlobsRequestInfo::default(); - self.range_blocks_and_blobs_requests.insert( - id, - BlocksAndBlobsByRangeRequest { - chain_id, - batch_id, - block_blob_info, - }, - ); - Ok(id) - } + }), + request_id: RequestId::Sync(SyncRequestId::RangeBlockAndBlobs { id }), + })?; } + + Ok(id) } - /// A blocks by range request sent by the backfill sync algorithm - pub fn backfill_blocks_by_range_request( + /// A blocks by range request sent by the range sync algorithm + pub fn blocks_and_blobs_by_range_request( &mut self, peer_id: PeerId, batch_type: ByRangeRequestType, request: BlocksByRangeRequest, - batch_id: BatchId, + sender_id: RangeRequestId, ) -> Result { - match batch_type { - ByRangeRequestType::Blocks => { - trace!( - self.log, - "Sending backfill BlocksByRange request"; - "method" => "BlocksByRange", - "count" => request.count(), - "peer" => %peer_id, - ); - let request = Request::BlocksByRange(request); - let id = self.next_id(); - let request_id = RequestId::Sync(SyncRequestId::BackFillBlocks { id }); - self.send_network_msg(NetworkMessage::SendRequest { - peer_id, - request, - request_id, - })?; - self.backfill_requests.insert(id, batch_id); - Ok(id) - } - ByRangeRequestType::BlocksAndBlobs => { - debug!( - self.log, - "Sending backfill BlocksByRange and BlobsByRange requests"; - "method" => "Mixed by range request", - "count" => request.count(), - "peer" => %peer_id, - ); - - // create the shared request id. This is fine since the rpc handles substream ids. - let id = self.next_id(); - let request_id = RequestId::Sync(SyncRequestId::BackFillBlockAndBlobs { id }); - - // Create the blob request based on the blob request. - let blobs_request = Request::BlobsByRange(BlobsByRangeRequest { - start_slot: *request.start_slot(), - count: *request.count(), - }); - let blocks_request = Request::BlocksByRange(request); - - // Send both requests. Make sure both can be sent. - self.send_network_msg(NetworkMessage::SendRequest { - peer_id, - request: blocks_request, - request_id, - })?; - self.send_network_msg(NetworkMessage::SendRequest { - peer_id, - request: blobs_request, - request_id, - })?; - let block_blob_info = BlocksAndBlobsRequestInfo::default(); - self.backfill_blocks_and_blobs_requests - .insert(id, (batch_id, block_blob_info)); - Ok(id) - } - } - } - - /// Response for a request that is only for blocks. - pub fn range_sync_block_only_response( - &mut self, - request_id: Id, - is_stream_terminator: bool, - ) -> Option<(ChainId, BatchId)> { - if is_stream_terminator { - self.range_requests.remove(&request_id) - } else { - self.range_requests.get(&request_id).copied() - } - } - - /// Received a blocks by range response for a request that couples blocks and blobs. - pub fn range_sync_block_and_blob_response( - &mut self, - request_id: Id, - block_or_blob: BlockOrBlob, - ) -> Option<(ChainId, BlocksAndBlobsByRangeResponse)> { - match self.range_blocks_and_blobs_requests.entry(request_id) { - Entry::Occupied(mut entry) => { - let req = entry.get_mut(); - let info = &mut req.block_blob_info; - match block_or_blob { - BlockOrBlob::Block(maybe_block) => info.add_block_response(maybe_block), - BlockOrBlob::Blob(maybe_sidecar) => info.add_sidecar_response(maybe_sidecar), - } - if info.is_finished() { - // If the request is finished, dequeue everything - let BlocksAndBlobsByRangeRequest { - chain_id, - batch_id, - block_blob_info, - } = entry.remove(); - Some(( - chain_id, - BlocksAndBlobsByRangeResponse { - batch_id, - responses: block_blob_info.into_responses(), - }, - )) - } else { - None - } - } - Entry::Vacant(_) => None, - } + let id = self.blocks_by_range_request(peer_id, batch_type, request)?; + self.range_blocks_and_blobs_requests + .insert(id, (sender_id, BlocksAndBlobsRequestInfo::new(batch_type))); + Ok(id) } - pub fn range_sync_request_failed( - &mut self, - request_id: Id, - batch_type: ByRangeRequestType, - ) -> Option<(ChainId, BatchId)> { - let req = match batch_type { - ByRangeRequestType::BlocksAndBlobs => self - .range_blocks_and_blobs_requests - .remove(&request_id) - .map(|req| (req.chain_id, req.batch_id)), - ByRangeRequestType::Blocks => self.range_requests.remove(&request_id), - }; - if let Some(req) = req { + pub fn range_request_failed(&mut self, request_id: Id) -> Option { + let sender_id = self + .range_blocks_and_blobs_requests + .remove(&request_id) + .map(|(sender_id, _info)| sender_id); + if let Some(sender_id) = sender_id { debug!( self.log, - "Range sync request failed"; + "Sync range request failed"; "request_id" => request_id, - "batch_type" => ?batch_type, - "chain_id" => ?req.0, - "batch_id" => ?req.1 + "sender_id" => ?sender_id ); - Some(req) + Some(sender_id) } else { - debug!(self.log, "Range sync request failed"; "request_id" => request_id, "batch_type" => ?batch_type); + debug!(self.log, "Sync range request failed"; "request_id" => request_id); None } } - pub fn backfill_request_failed( - &mut self, - request_id: Id, - batch_type: ByRangeRequestType, - ) -> Option { - let batch_id = match batch_type { - ByRangeRequestType::BlocksAndBlobs => self - .backfill_blocks_and_blobs_requests - .remove(&request_id) - .map(|(batch_id, _info)| batch_id), - ByRangeRequestType::Blocks => self.backfill_requests.remove(&request_id), - }; - if let Some(batch_id) = batch_id { - debug!( - self.log, - "Backfill sync request failed"; - "request_id" => request_id, - "batch_type" => ?batch_type, - "batch_id" => ?batch_id - ); - Some(batch_id) - } else { - debug!(self.log, "Backfill sync request failed"; "request_id" => request_id, "batch_type" => ?batch_type); - None - } - } - - /// Response for a request that is only for blocks. - pub fn backfill_sync_only_blocks_response( - &mut self, - request_id: Id, - is_stream_terminator: bool, - ) -> Option { - if is_stream_terminator { - self.backfill_requests.remove(&request_id) - } else { - self.backfill_requests.get(&request_id).copied() - } - } - /// Received a blocks by range or blobs by range response for a request that couples blocks ' /// and blobs. - pub fn backfill_sync_block_and_blob_response( + pub fn range_block_and_blob_response( &mut self, request_id: Id, block_or_blob: BlockOrBlob, ) -> Option> { - match self.backfill_blocks_and_blobs_requests.entry(request_id) { + match self.range_blocks_and_blobs_requests.entry(request_id) { Entry::Occupied(mut entry) => { let (_, info) = entry.get_mut(); match block_or_blob { @@ -416,12 +229,12 @@ impl SyncNetworkContext { } if info.is_finished() { // If the request is finished, dequeue everything - let (batch_id, info) = entry.remove(); - - let responses = info.into_responses(); + let (sender_id, info) = entry.remove(); + let request_type = info.get_request_type(); Some(BlocksAndBlobsByRangeResponse { - batch_id, - responses, + sender_id, + request_type, + responses: info.into_responses(), }) } else { None @@ -586,18 +399,10 @@ impl SyncNetworkContext { pub fn insert_range_blocks_and_blobs_request( &mut self, id: Id, - request: BlocksAndBlobsByRangeRequest, - ) { - self.range_blocks_and_blobs_requests.insert(id, request); - } - - pub fn insert_backfill_blocks_and_blobs_requests( - &mut self, - id: Id, - batch_id: BatchId, - request: BlocksAndBlobsRequestInfo, + sender_id: RangeRequestId, + info: BlocksAndBlobsRequestInfo, ) { - self.backfill_blocks_and_blobs_requests - .insert(id, (batch_id, request)); + self.range_blocks_and_blobs_requests + .insert(id, (sender_id, info)); } } diff --git a/beacon_node/network/src/sync/range_sync/chain.rs b/beacon_node/network/src/sync/range_sync/chain.rs index 5a77340e3b5..c60cdb2cc9f 100644 --- a/beacon_node/network/src/sync/range_sync/chain.rs +++ b/beacon_node/network/src/sync/range_sync/chain.rs @@ -1,5 +1,6 @@ use super::batch::{BatchInfo, BatchProcessingResult, BatchState}; use crate::network_beacon_processor::ChainSegmentProcessId; +use crate::sync::network_context::RangeRequestId; use crate::sync::{ manager::Id, network_context::SyncNetworkContext, BatchOperationOutcome, BatchProcessResult, }; @@ -905,7 +906,15 @@ impl SyncingChain { ) -> ProcessingResult { if let Some(batch) = self.batches.get_mut(&batch_id) { let (request, batch_type) = batch.to_blocks_by_range_request(); - match network.blocks_by_range_request(peer, batch_type, request, self.id, batch_id) { + match network.blocks_and_blobs_by_range_request( + peer, + batch_type, + request, + RangeRequestId::RangeSync { + chain_id: self.id, + batch_id, + }, + ) { Ok(request_id) => { // inform the batch about the new request batch.start_downloading_from_peer(peer, request_id)?; diff --git a/beacon_node/network/src/sync/range_sync/range.rs b/beacon_node/network/src/sync/range_sync/range.rs index a159eb45410..c8e82666840 100644 --- a/beacon_node/network/src/sync/range_sync/range.rs +++ b/beacon_node/network/src/sync/range_sync/range.rs @@ -384,7 +384,7 @@ mod tests { use crate::NetworkMessage; use super::*; - use crate::sync::network_context::BlockOrBlob; + use crate::sync::network_context::{BlockOrBlob, RangeRequestId}; use beacon_chain::builder::Witness; use beacon_chain::eth1_chain::CachingEth1Backend; use beacon_chain::parking_lot::RwLock; @@ -548,6 +548,51 @@ mod tests { (block_req_id, blob_req_id) } + fn complete_range_block_and_blobs_response( + &mut self, + block_req: RequestId, + blob_req_opt: Option, + ) -> (ChainId, BatchId, Id) { + if blob_req_opt.is_some() { + match block_req { + RequestId::Sync(crate::sync::manager::RequestId::RangeBlockAndBlobs { id }) => { + let _ = self + .cx + .range_block_and_blob_response(id, BlockOrBlob::Block(None)); + let response = self + .cx + .range_block_and_blob_response(id, BlockOrBlob::Blob(None)) + .unwrap(); + let (chain_id, batch_id) = + TestRig::unwrap_range_request_id(response.sender_id); + (chain_id, batch_id, id) + } + other => panic!("unexpected request {:?}", other), + } + } else { + match block_req { + RequestId::Sync(crate::sync::manager::RequestId::RangeBlockAndBlobs { id }) => { + let response = self + .cx + .range_block_and_blob_response(id, BlockOrBlob::Block(None)) + .unwrap(); + let (chain_id, batch_id) = + TestRig::unwrap_range_request_id(response.sender_id); + (chain_id, batch_id, id) + } + other => panic!("unexpected request {:?}", other), + } + } + } + + fn unwrap_range_request_id(sender_id: RangeRequestId) -> (ChainId, BatchId) { + if let RangeRequestId::RangeSync { chain_id, batch_id } = sender_id { + (chain_id, batch_id) + } else { + panic!("expected RangeSync request: {:?}", sender_id) + } + } + /// Produce a head peer fn head_peer( &self, @@ -744,29 +789,8 @@ mod tests { range.add_peer(&mut rig.cx, local_info, peer1, head_info); let (block_req, blob_req_opt) = rig.grab_request(&peer1, fork); - let (chain1, batch1, id1) = if blob_req_opt.is_some() { - match block_req { - RequestId::Sync(crate::sync::manager::RequestId::RangeBlockAndBlobs { id }) => { - let _ = rig - .cx - .range_sync_block_and_blob_response(id, BlockOrBlob::Block(None)); - let (chain1, response) = rig - .cx - .range_sync_block_and_blob_response(id, BlockOrBlob::Blob(None)) - .unwrap(); - (chain1, response.batch_id, id) - } - other => panic!("unexpected request {:?}", other), - } - } else { - match block_req { - RequestId::Sync(crate::sync::manager::RequestId::RangeBlocks { id }) => { - let (chain, batch) = rig.cx.range_sync_block_only_response(id, true).unwrap(); - (chain, batch, id) - } - other => panic!("unexpected request {:?}", other), - } - }; + let (chain1, batch1, id1) = + rig.complete_range_block_and_blobs_response(block_req, blob_req_opt); // make the ee offline rig.cx.update_execution_engine_state(EngineState::Offline); @@ -782,29 +806,8 @@ mod tests { range.add_peer(&mut rig.cx, local_info, peer2, finalized_info); let (block_req, blob_req_opt) = rig.grab_request(&peer2, fork); - let (chain2, batch2, id2) = if blob_req_opt.is_some() { - match block_req { - RequestId::Sync(crate::sync::manager::RequestId::RangeBlockAndBlobs { id }) => { - let _ = rig - .cx - .range_sync_block_and_blob_response(id, BlockOrBlob::Block(None)); - let (chain2, response) = rig - .cx - .range_sync_block_and_blob_response(id, BlockOrBlob::Blob(None)) - .unwrap(); - (chain2, response.batch_id, id) - } - other => panic!("unexpected request {:?}", other), - } - } else { - match block_req { - RequestId::Sync(crate::sync::manager::RequestId::RangeBlocks { id }) => { - let (chain, batch) = rig.cx.range_sync_block_only_response(id, true).unwrap(); - (chain, batch, id) - } - other => panic!("unexpected request {:?}", other), - } - }; + let (chain2, batch2, id2) = + rig.complete_range_block_and_blobs_response(block_req, blob_req_opt); // send the response to the request range.blocks_by_range_response(&mut rig.cx, peer2, chain2, batch2, id2, None); From 116a55e8a501bbc260b29a093039aa8f74ddb6aa Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Sat, 13 Apr 2024 01:15:04 +0900 Subject: [PATCH 03/37] Ensure proper ReqResp response stream termination (#5556) * Ensure proper ReqResp response stream termination * Update beacon_node/network/src/network_beacon_processor/rpc_methods.rs * Update beacon_node/network/src/network_beacon_processor/rpc_methods.rs * cargo fmt --- .../network_beacon_processor/rpc_methods.rs | 264 ++++++++++-------- 1 file changed, 149 insertions(+), 115 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs index 1b4b93adb0c..2a00484803a 100644 --- a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs @@ -212,6 +212,7 @@ impl NetworkBeaconProcessor { "load_blocks_by_root_blocks", ) } + /// Handle a `BlobsByRoot` request from the peer. pub fn handle_blobs_by_root_request( self: Arc, @@ -219,10 +220,25 @@ impl NetworkBeaconProcessor { request_id: PeerRequestId, request: BlobsByRootRequest, ) { + self.terminate_response_stream( + peer_id, + request_id, + self.handle_blobs_by_root_request_inner(peer_id, request_id, request), + Response::BlobsByRoot, + ); + } + + /// Handle a `BlobsByRoot` request from the peer. + pub fn handle_blobs_by_root_request_inner( + &self, + peer_id: PeerId, + request_id: PeerRequestId, + request: BlobsByRootRequest, + ) -> Result<(), (RPCResponseErrorCode, &'static str)> { let Some(requested_root) = request.blob_ids.as_slice().first().map(|id| id.block_root) else { // No blob ids requested. - return; + return Ok(()); }; let requested_indices = request .blob_ids @@ -231,7 +247,6 @@ impl NetworkBeaconProcessor { .map(|id| id.index) .collect::>(); let mut send_blob_count = 0; - let send_response = true; let mut blob_list_results = HashMap::new(); for id in request.blob_ids.as_slice() { @@ -287,10 +302,7 @@ impl NetworkBeaconProcessor { "returned" => send_blob_count ); - // send stream termination - if send_response { - self.send_response(peer_id, Response::BlobsByRoot(None), request_id); - } + Ok(()) } /// Handle a `LightClientBootstrap` request from the peer. @@ -300,33 +312,29 @@ impl NetworkBeaconProcessor { request_id: PeerRequestId, request: LightClientBootstrapRequest, ) { - let block_root = request.root; - match self.chain.get_light_client_bootstrap(&block_root) { - Ok(Some((bootstrap, _))) => self.send_response( - peer_id, - Response::LightClientBootstrap(Arc::new(bootstrap)), - request_id, - ), - Ok(None) => self.send_error_response( - peer_id, - RPCResponseErrorCode::ResourceUnavailable, - "Bootstrap not available".into(), - request_id, - ), - Err(e) => { - self.send_error_response( - peer_id, + self.terminate_response_single_item( + peer_id, + request_id, + match self.chain.get_light_client_bootstrap(&request.root) { + Ok(Some((bootstrap, _))) => Ok(Arc::new(bootstrap)), + Ok(None) => Err(( RPCResponseErrorCode::ResourceUnavailable, - "Bootstrap not available".into(), - request_id, - ); - error!(self.log, "Error getting LightClientBootstrap instance"; - "block_root" => ?block_root, - "peer" => %peer_id, - "error" => ?e - ) - } - }; + "Bootstrap not available", + )), + Err(e) => { + error!(self.log, "Error getting LightClientBootstrap instance"; + "block_root" => ?request.root, + "peer" => %peer_id, + "error" => ?e + ); + Err(( + RPCResponseErrorCode::ResourceUnavailable, + "Bootstrap not available", + )) + } + }, + Response::LightClientBootstrap, + ); } /// Handle a `LightClientOptimisticUpdate` request from the peer. @@ -335,25 +343,22 @@ impl NetworkBeaconProcessor { peer_id: PeerId, request_id: PeerRequestId, ) { - let Some(light_client_optimistic_update) = self - .chain - .light_client_server_cache - .get_latest_optimistic_update() - else { - self.send_error_response( - peer_id, - RPCResponseErrorCode::ResourceUnavailable, - "Latest optimistic update not available".into(), - request_id, - ); - return; - }; - - self.send_response( + self.terminate_response_single_item( peer_id, - Response::LightClientOptimisticUpdate(Arc::new(light_client_optimistic_update)), request_id, - ) + match self + .chain + .light_client_server_cache + .get_latest_optimistic_update() + { + Some(update) => Ok(Arc::new(update)), + None => Err(( + RPCResponseErrorCode::ResourceUnavailable, + "Latest optimistic update not available", + )), + }, + Response::LightClientOptimisticUpdate, + ); } /// Handle a `LightClientFinalityUpdate` request from the peer. @@ -362,25 +367,22 @@ impl NetworkBeaconProcessor { peer_id: PeerId, request_id: PeerRequestId, ) { - let Some(light_client_finality_update) = self - .chain - .light_client_server_cache - .get_latest_finality_update() - else { - self.send_error_response( - peer_id, - RPCResponseErrorCode::ResourceUnavailable, - "Latest finality update not available".into(), - request_id, - ); - return; - }; - - self.send_response( + self.terminate_response_single_item( peer_id, - Response::LightClientFinalityUpdate(Arc::new(light_client_finality_update)), request_id, - ) + match self + .chain + .light_client_server_cache + .get_latest_finality_update() + { + Some(update) => Ok(Arc::new(update)), + None => Err(( + RPCResponseErrorCode::ResourceUnavailable, + "Latest finality update not available", + )), + }, + Response::LightClientFinalityUpdate, + ); } /// Handle a `BlocksByRange` request from the peer. @@ -637,6 +639,21 @@ impl NetworkBeaconProcessor { request_id: PeerRequestId, req: BlobsByRangeRequest, ) { + self.terminate_response_stream( + peer_id, + request_id, + self.handle_blobs_by_range_request_inner(peer_id, request_id, req), + Response::BlobsByRange, + ); + } + + /// Handle a `BlobsByRange` request from the peer. + fn handle_blobs_by_range_request_inner( + &self, + peer_id: PeerId, + request_id: PeerRequestId, + req: BlobsByRangeRequest, + ) -> Result<(), (RPCResponseErrorCode, &'static str)> { debug!(self.log, "Received BlobsByRange Request"; "peer_id" => %peer_id, "count" => req.count, @@ -645,12 +662,10 @@ impl NetworkBeaconProcessor { // Should not send more than max request blocks if req.max_blobs_requested::() > self.chain.spec.max_request_blob_sidecars { - return self.send_error_response( - peer_id, + return Err(( RPCResponseErrorCode::InvalidRequest, - "Request exceeded `MAX_REQUEST_BLOBS_SIDECARS`".into(), - request_id, - ); + "Request exceeded `MAX_REQUEST_BLOBS_SIDECARS`", + )); } let request_start_slot = Slot::from(req.start_slot); @@ -659,13 +674,10 @@ impl NetworkBeaconProcessor { Some(boundary) => boundary.start_slot(T::EthSpec::slots_per_epoch()), None => { debug!(self.log, "Deneb fork is disabled"); - self.send_error_response( - peer_id, + return Err(( RPCResponseErrorCode::InvalidRequest, - "Deneb fork is disabled".into(), - request_id, - ); - return; + "Deneb fork is disabled", + )); } }; @@ -685,19 +697,15 @@ impl NetworkBeaconProcessor { ); return if data_availability_boundary_slot < oldest_blob_slot { - self.send_error_response( - peer_id, + Err(( RPCResponseErrorCode::ResourceUnavailable, - "blobs pruned within boundary".into(), - request_id, - ) + "blobs pruned within boundary", + )) } else { - self.send_error_response( - peer_id, + Err(( RPCResponseErrorCode::InvalidRequest, - "Req outside availability period".into(), - request_id, - ) + "Req outside availability period", + )) }; } @@ -714,25 +722,15 @@ impl NetworkBeaconProcessor { "requested_slot" => slot, "oldest_known_slot" => oldest_block_slot ); - return self.send_error_response( - peer_id, - RPCResponseErrorCode::ResourceUnavailable, - "Backfilling".into(), - request_id, - ); + return Err((RPCResponseErrorCode::ResourceUnavailable, "Backfilling")); } Err(e) => { - self.send_error_response( - peer_id, - RPCResponseErrorCode::ServerError, - "Database error".into(), - request_id, - ); - return error!(self.log, "Unable to obtain root iter"; + error!(self.log, "Unable to obtain root iter"; "request" => ?req, "peer" => %peer_id, "error" => ?e ); + return Err((RPCResponseErrorCode::ServerError, "Database error")); } }; @@ -764,11 +762,12 @@ impl NetworkBeaconProcessor { let block_roots = match maybe_block_roots { Ok(block_roots) => block_roots, Err(e) => { - return error!(self.log, "Error during iteration over blocks"; + error!(self.log, "Error during iteration over blocks"; "request" => ?req, "peer" => %peer_id, "error" => ?e - ) + ); + return Err((RPCResponseErrorCode::ServerError, "Database error")); } }; @@ -776,7 +775,6 @@ impl NetworkBeaconProcessor { let block_roots = block_roots.into_iter().flatten(); let mut blobs_sent = 0; - let mut send_response = true; for root in block_roots { match self.chain.get_blobs(&root) { @@ -799,14 +797,10 @@ impl NetworkBeaconProcessor { "block_root" => ?root, "error" => ?e ); - self.send_error_response( - peer_id, + return Err(( RPCResponseErrorCode::ServerError, - "No blobs and failed fetching corresponding block".into(), - request_id, - ); - send_response = false; - break; + "No blobs and failed fetching corresponding block", + )); } } } @@ -826,13 +820,53 @@ impl NetworkBeaconProcessor { "returned" => blobs_sent ); - if send_response { - // send the stream terminator - self.send_network_message(NetworkMessage::SendResponse { + Ok(()) + } + + /// Helper function to ensure single item protocol always end with either a single chunk or an + /// error + fn terminate_response_single_item Response>( + &self, + peer_id: PeerId, + request_id: PeerRequestId, + result: Result, + into_response: F, + ) { + match result { + Ok(resp) => { + // Not necessary to explicitly send a termination message if this InboundRequest + // returns <= 1 for InboundRequest::expected_responses + // https://github.com/sigp/lighthouse/blob/3058b96f2560f1da04ada4f9d8ba8e5651794ff6/beacon_node/lighthouse_network/src/rpc/handler.rs#L555-L558 + self.send_network_message(NetworkMessage::SendResponse { + peer_id, + response: into_response(resp), + id: request_id, + }); + } + Err((error_code, reason)) => { + self.send_error_response(peer_id, error_code, reason.into(), request_id); + } + } + } + + /// Helper function to ensure streamed protocols with multiple responses always end with either + /// a stream termination or an error + fn terminate_response_stream) -> Response>( + &self, + peer_id: PeerId, + request_id: PeerRequestId, + result: Result<(), (RPCResponseErrorCode, &'static str)>, + into_response: F, + ) { + match result { + Ok(_) => self.send_network_message(NetworkMessage::SendResponse { peer_id, - response: Response::BlobsByRange(None), + response: into_response(None), id: request_id, - }); + }), + Err((error_code, reason)) => { + self.send_error_response(peer_id, error_code, reason.into(), request_id); + } } } } From b6a1c863a23892255f4c95f61fb2583282879bbc Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Sat, 13 Apr 2024 04:30:04 +0900 Subject: [PATCH 04/37] Use spawn_async in ByRoot handling workers (#5557) * Use spawn_async in ByRoot handling workers * box large variants --- beacon_node/beacon_processor/src/lib.rs | 35 +- .../src/network_beacon_processor/mod.rs | 36 +- .../network_beacon_processor/rpc_methods.rs | 342 ++++++++---------- 3 files changed, 179 insertions(+), 234 deletions(-) diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index 9dd72845180..9b83e9cacbc 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -571,7 +571,7 @@ pub enum BlockingOrAsync { /// queuing specifics. pub enum Work { GossipAttestation { - attestation: GossipAttestationPackage, + attestation: Box>, process_individual: Box) + Send + Sync>, process_batch: Box>) + Send + Sync>, }, @@ -583,7 +583,7 @@ pub enum Work { process_batch: Box>) + Send + Sync>, }, GossipAggregate { - aggregate: GossipAggregatePackage, + aggregate: Box>, process_individual: Box) + Send + Sync>, process_batch: Box>) + Send + Sync>, }, @@ -624,8 +624,8 @@ pub enum Work { ChainSegment(AsyncFn), ChainSegmentBackfill(AsyncFn), Status(BlockingFn), - BlocksByRangeRequest(BlockingFnWithManualSendOnIdle), - BlocksByRootsRequest(BlockingFnWithManualSendOnIdle), + BlocksByRangeRequest(AsyncFn), + BlocksByRootsRequest(AsyncFn), BlobsByRangeRequest(BlockingFn), BlobsByRootsRequest(BlockingFn), GossipBlsToExecutionChange(BlockingFn), @@ -1015,7 +1015,7 @@ impl BeaconProcessor { process_individual: _, process_batch, } => { - aggregates.push(aggregate); + aggregates.push(*aggregate); if process_batch_opt.is_none() { process_batch_opt = Some(process_batch); } @@ -1075,7 +1075,7 @@ impl BeaconProcessor { process_individual: _, process_batch, } => { - attestations.push(attestation); + attestations.push(*attestation); if process_batch_opt.is_none() { process_batch_opt = Some(process_batch); } @@ -1445,7 +1445,7 @@ impl BeaconProcessor { process_individual, process_batch: _, } => task_spawner.spawn_blocking(move || { - process_individual(attestation); + process_individual(*attestation); }), Work::GossipAttestationBatch { attestations, @@ -1458,7 +1458,7 @@ impl BeaconProcessor { process_individual, process_batch: _, } => task_spawner.spawn_blocking(move || { - process_individual(aggregate); + process_individual(*aggregate); }), Work::GossipAggregateBatch { aggregates, @@ -1493,7 +1493,7 @@ impl BeaconProcessor { task_spawner.spawn_blocking(process_fn) } Work::BlocksByRangeRequest(work) | Work::BlocksByRootsRequest(work) => { - task_spawner.spawn_blocking_with_manual_send_idle(work) + task_spawner.spawn_async(work) } Work::ChainSegmentBackfill(process_fn) => task_spawner.spawn_async(process_fn), Work::ApiRequestP0(process_fn) | Work::ApiRequestP1(process_fn) => match process_fn { @@ -1555,23 +1555,6 @@ impl TaskSpawner { WORKER_TASK_NAME, ) } - - /// Spawn a blocking task, passing the `SendOnDrop` into the task. - /// - /// ## Notes - /// - /// Users must ensure the `SendOnDrop` is dropped at the appropriate time! - pub fn spawn_blocking_with_manual_send_idle(self, task: F) - where - F: FnOnce(SendOnDrop) + Send + 'static, - { - self.executor.spawn_blocking( - || { - task(self.send_idle_on_drop); - }, - WORKER_TASK_NAME, - ) - } } /// This struct will send a message on `self.tx` when it is dropped. An error will be logged on diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 6872a712c9d..27b9e676da6 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -102,14 +102,14 @@ impl NetworkBeaconProcessor { self.try_send(BeaconWorkEvent { drop_during_sync: true, work: Work::GossipAttestation { - attestation: GossipAttestationPackage { + attestation: Box::new(GossipAttestationPackage { message_id, peer_id, attestation: Box::new(attestation), subnet_id, should_import, seen_timestamp, - }, + }), process_individual: Box::new(process_individual), process_batch: Box::new(process_batch), }, @@ -148,13 +148,13 @@ impl NetworkBeaconProcessor { self.try_send(BeaconWorkEvent { drop_during_sync: true, work: Work::GossipAggregate { - aggregate: GossipAggregatePackage { + aggregate: Box::new(GossipAggregatePackage { message_id, peer_id, aggregate: Box::new(aggregate), beacon_block_root, seen_timestamp, - }, + }), process_individual: Box::new(process_individual), process_batch: Box::new(process_batch), }, @@ -508,20 +508,16 @@ impl NetworkBeaconProcessor { request: BlocksByRangeRequest, ) -> Result<(), Error> { let processor = self.clone(); - let process_fn = move |send_idle_on_drop| { + let process_fn = async move { let executor = processor.executor.clone(); - processor.handle_blocks_by_range_request( - executor, - send_idle_on_drop, - peer_id, - request_id, - request, - ) + processor + .handle_blocks_by_range_request(executor, peer_id, request_id, request) + .await; }; self.try_send(BeaconWorkEvent { drop_during_sync: false, - work: Work::BlocksByRangeRequest(Box::new(process_fn)), + work: Work::BlocksByRangeRequest(Box::pin(process_fn)), }) } @@ -533,20 +529,16 @@ impl NetworkBeaconProcessor { request: BlocksByRootRequest, ) -> Result<(), Error> { let processor = self.clone(); - let process_fn = move |send_idle_on_drop| { + let process_fn = async move { let executor = processor.executor.clone(); - processor.handle_blocks_by_root_request( - executor, - send_idle_on_drop, - peer_id, - request_id, - request, - ) + processor + .handle_blocks_by_root_request(executor, peer_id, request_id, request) + .await; }; self.try_send(BeaconWorkEvent { drop_during_sync: false, - work: Work::BlocksByRootsRequest(Box::new(process_fn)), + work: Work::BlocksByRootsRequest(Box::pin(process_fn)), }) } diff --git a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs index 2a00484803a..81c8f662ee9 100644 --- a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs @@ -3,7 +3,6 @@ use crate::service::NetworkMessage; use crate::status::ToStatusMessage; use crate::sync::SyncMessage; use beacon_chain::{BeaconChainError, BeaconChainTypes, HistoricalBlockError, WhenSlotSkipped}; -use beacon_processor::SendOnDrop; use itertools::process_results; use lighthouse_network::rpc::methods::{BlobsByRangeRequest, BlobsByRootRequest}; use lighthouse_network::rpc::*; @@ -128,10 +127,9 @@ impl NetworkBeaconProcessor { } /// Handle a `BlocksByRoot` request from the peer. - pub fn handle_blocks_by_root_request( + pub async fn handle_blocks_by_root_request( self: Arc, executor: TaskExecutor, - send_on_drop: SendOnDrop, peer_id: PeerId, request_id: PeerRequestId, request: BlocksByRootRequest, @@ -145,72 +143,61 @@ impl NetworkBeaconProcessor { Err(e) => return error!(self.log, "Error getting block stream"; "error" => ?e), }; // Fetching blocks is async because it may have to hit the execution layer for payloads. - executor.spawn( - async move { - let mut send_block_count = 0; - let mut send_response = true; - while let Some((root, result)) = block_stream.next().await { - match result.as_ref() { - Ok(Some(block)) => { - self.send_response( - peer_id, - Response::BlocksByRoot(Some(block.clone())), - request_id, - ); - send_block_count += 1; - } - Ok(None) => { - debug!( - self.log, - "Peer requested unknown block"; - "peer" => %peer_id, - "request_root" => ?root - ); - } - Err(BeaconChainError::BlockHashMissingFromExecutionLayer(_)) => { - debug!( - self.log, - "Failed to fetch execution payload for blocks by root request"; - "block_root" => ?root, - "reason" => "execution layer not synced", - ); - // send the stream terminator - self.send_error_response( - peer_id, - RPCResponseErrorCode::ResourceUnavailable, - "Execution layer not synced".into(), - request_id, - ); - send_response = false; - break; - } - Err(e) => { - debug!( - self.log, - "Error fetching block for peer"; - "peer" => %peer_id, - "request_root" => ?root, - "error" => ?e, - ); - } - } + let mut send_block_count = 0; + while let Some((root, result)) = block_stream.next().await { + match result.as_ref() { + Ok(Some(block)) => { + self.send_response( + peer_id, + Response::BlocksByRoot(Some(block.clone())), + request_id, + ); + send_block_count += 1; } - debug!( - self.log, - "Received BlocksByRoot Request"; - "peer" => %peer_id, - "requested" => requested_blocks, - "returned" => %send_block_count - ); - - // send stream termination - if send_response { - self.send_response(peer_id, Response::BlocksByRoot(None), request_id); + Ok(None) => { + debug!( + self.log, + "Peer requested unknown block"; + "peer" => %peer_id, + "request_root" => ?root + ); } - drop(send_on_drop); - }, - "load_blocks_by_root_blocks", - ) + Err(BeaconChainError::BlockHashMissingFromExecutionLayer(_)) => { + debug!( + self.log, + "Failed to fetch execution payload for blocks by root request"; + "block_root" => ?root, + "reason" => "execution layer not synced", + ); + // send the stream terminator + return self.send_error_response( + peer_id, + RPCResponseErrorCode::ResourceUnavailable, + "Execution layer not synced".into(), + request_id, + ); + } + Err(e) => { + debug!( + self.log, + "Error fetching block for peer"; + "peer" => %peer_id, + "request_root" => ?root, + "error" => ?e, + ); + } + } + } + debug!( + self.log, + "Received BlocksByRoot Request"; + "peer" => %peer_id, + "requested" => requested_blocks, + "returned" => %send_block_count + ); + + // send stream termination + self.send_response(peer_id, Response::BlocksByRoot(None), request_id); } /// Handle a `BlobsByRoot` request from the peer. @@ -386,10 +373,9 @@ impl NetworkBeaconProcessor { } /// Handle a `BlocksByRange` request from the peer. - pub fn handle_blocks_by_range_request( + pub async fn handle_blocks_by_range_request( self: Arc, executor: TaskExecutor, - send_on_drop: SendOnDrop, peer_id: PeerId, request_id: PeerRequestId, req: BlocksByRangeRequest, @@ -499,137 +485,121 @@ impl NetworkBeaconProcessor { }; // Fetching blocks is async because it may have to hit the execution layer for payloads. - executor.spawn( - async move { - let mut blocks_sent = 0; - let mut send_response = true; - - while let Some((root, result)) = block_stream.next().await { - match result.as_ref() { - Ok(Some(block)) => { - // Due to skip slots, blocks could be out of the range, we ensure they - // are in the range before sending - if block.slot() >= *req.start_slot() - && block.slot() < req.start_slot() + req.count() - { - blocks_sent += 1; - self.send_network_message(NetworkMessage::SendResponse { - peer_id, - response: Response::BlocksByRange(Some(block.clone())), - id: request_id, - }); - } - } - Ok(None) => { - error!( - self.log, - "Block in the chain is not in the store"; - "request" => ?req, - "peer" => %peer_id, - "request_root" => ?root - ); - self.send_error_response( - peer_id, - RPCResponseErrorCode::ServerError, - "Database inconsistency".into(), - request_id, - ); - send_response = false; - break; - } - Err(BeaconChainError::BlockHashMissingFromExecutionLayer(_)) => { - debug!( - self.log, - "Failed to fetch execution payload for blocks by range request"; - "block_root" => ?root, - "reason" => "execution layer not synced", - ); - // send the stream terminator - self.send_error_response( - peer_id, - RPCResponseErrorCode::ResourceUnavailable, - "Execution layer not synced".into(), - request_id, - ); - send_response = false; - break; - } - Err(e) => { - if matches!( - e, - BeaconChainError::ExecutionLayerErrorPayloadReconstruction(_block_hash, ref boxed_error) - if matches!(**boxed_error, execution_layer::Error::EngineError(_)) - ) { - warn!( - self.log, - "Error rebuilding payload for peer"; - "info" => "this may occur occasionally when the EE is busy", - "block_root" => ?root, - "error" => ?e, - ); - } else { - error!( - self.log, - "Error fetching block for peer"; - "block_root" => ?root, - "error" => ?e - ); - } - - // send the stream terminator - self.send_error_response( - peer_id, - RPCResponseErrorCode::ServerError, - "Failed fetching blocks".into(), - request_id, - ); - send_response = false; - break; - } + let mut blocks_sent = 0; + + while let Some((root, result)) = block_stream.next().await { + match result.as_ref() { + Ok(Some(block)) => { + // Due to skip slots, blocks could be out of the range, we ensure they + // are in the range before sending + if block.slot() >= *req.start_slot() + && block.slot() < req.start_slot() + req.count() + { + blocks_sent += 1; + self.send_network_message(NetworkMessage::SendResponse { + peer_id, + response: Response::BlocksByRange(Some(block.clone())), + id: request_id, + }); } } - - let current_slot = self - .chain - .slot() - .unwrap_or_else(|_| self.chain.slot_clock.genesis_slot()); - - if blocks_sent < (*req.count() as usize) { - debug!( + Ok(None) => { + error!( self.log, - "BlocksByRange outgoing response processed"; + "Block in the chain is not in the store"; + "request" => ?req, "peer" => %peer_id, - "msg" => "Failed to return all requested blocks", - "start_slot" => req.start_slot(), - "current_slot" => current_slot, - "requested" => req.count(), - "returned" => blocks_sent + "request_root" => ?root ); - } else { + return self.send_error_response( + peer_id, + RPCResponseErrorCode::ServerError, + "Database inconsistency".into(), + request_id, + ); + } + Err(BeaconChainError::BlockHashMissingFromExecutionLayer(_)) => { debug!( self.log, - "BlocksByRange outgoing response processed"; - "peer" => %peer_id, - "start_slot" => req.start_slot(), - "current_slot" => current_slot, - "requested" => req.count(), - "returned" => blocks_sent + "Failed to fetch execution payload for blocks by range request"; + "block_root" => ?root, + "reason" => "execution layer not synced", + ); + // send the stream terminator + return self.send_error_response( + peer_id, + RPCResponseErrorCode::ResourceUnavailable, + "Execution layer not synced".into(), + request_id, ); } + Err(e) => { + if matches!( + e, + BeaconChainError::ExecutionLayerErrorPayloadReconstruction(_block_hash, ref boxed_error) + if matches!(**boxed_error, execution_layer::Error::EngineError(_)) + ) { + warn!( + self.log, + "Error rebuilding payload for peer"; + "info" => "this may occur occasionally when the EE is busy", + "block_root" => ?root, + "error" => ?e, + ); + } else { + error!( + self.log, + "Error fetching block for peer"; + "block_root" => ?root, + "error" => ?e + ); + } - if send_response { // send the stream terminator - self.send_network_message(NetworkMessage::SendResponse { + return self.send_error_response( peer_id, - response: Response::BlocksByRange(None), - id: request_id, - }); + RPCResponseErrorCode::ServerError, + "Failed fetching blocks".into(), + request_id, + ); } + } + } - drop(send_on_drop); - }, - "load_blocks_by_range_blocks", - ); + let current_slot = self + .chain + .slot() + .unwrap_or_else(|_| self.chain.slot_clock.genesis_slot()); + + if blocks_sent < (*req.count() as usize) { + debug!( + self.log, + "BlocksByRange outgoing response processed"; + "peer" => %peer_id, + "msg" => "Failed to return all requested blocks", + "start_slot" => req.start_slot(), + "current_slot" => current_slot, + "requested" => req.count(), + "returned" => blocks_sent + ); + } else { + debug!( + self.log, + "BlocksByRange outgoing response processed"; + "peer" => %peer_id, + "start_slot" => req.start_slot(), + "current_slot" => current_slot, + "requested" => req.count(), + "returned" => blocks_sent + ); + } + + // send the stream terminator + self.send_network_message(NetworkMessage::SendResponse { + peer_id, + response: Response::BlocksByRange(None), + id: request_id, + }); } /// Handle a `BlobsByRange` request from the peer. From fee2ee9c08c15d98f5a96892b44f833977cd0f24 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 15 Apr 2024 22:47:08 +0900 Subject: [PATCH 05/37] Make SingleLookupRequestState fields private (#5563) * Make SingleLookupRequestState fields private * Fix tests --- .../network/src/sync/block_lookups/common.rs | 98 +++++------ .../network/src/sync/block_lookups/mod.rs | 74 ++++----- .../src/sync/block_lookups/parent_lookup.rs | 28 ++-- .../sync/block_lookups/single_block_lookup.rs | 157 +++++++++++++----- beacon_node/network/src/sync/manager.rs | 1 + 5 files changed, 200 insertions(+), 158 deletions(-) diff --git a/beacon_node/network/src/sync/block_lookups/common.rs b/beacon_node/network/src/sync/block_lookups/common.rs index a46e7d09376..8f7881eea8a 100644 --- a/beacon_node/network/src/sync/block_lookups/common.rs +++ b/beacon_node/network/src/sync/block_lookups/common.rs @@ -12,7 +12,6 @@ use beacon_chain::data_availability_checker::ChildComponents; use beacon_chain::{get_block_root, BeaconChainTypes}; use lighthouse_network::rpc::methods::BlobsByRootRequest; use lighthouse_network::rpc::BlocksByRootRequest; -use rand::prelude::IteratorRandom; use std::ops::IndexMut; use std::sync::Arc; use std::time::Duration; @@ -104,7 +103,7 @@ pub trait RequestState { cx: &SyncNetworkContext, ) -> Result<(), LookupRequestError> { // Check if request is necessary. - if !matches!(self.get_state().state, State::AwaitingDownload) { + if !self.get_state().is_awaiting_download() { return Ok(()); } @@ -112,13 +111,12 @@ pub trait RequestState { let (peer_id, request) = self.build_request(&cx.chain.spec)?; // Update request state. - self.get_state_mut().state = State::Downloading { peer_id }; - self.get_state_mut().req_counter += 1; + let req_counter = self.get_state_mut().on_download_start(peer_id); // Make request let id = SingleLookupReqId { id, - req_counter: self.get_state().req_counter, + req_counter, lookup_type: L::lookup_type(), }; Self::make_request(id, peer_id, request, cx) @@ -130,8 +128,7 @@ pub trait RequestState { let request_state = self.get_state(); if request_state.failed_attempts() >= max_attempts { - let cannot_process = - request_state.failed_processing >= request_state.failed_downloading; + let cannot_process = request_state.more_failed_processing_attempts(); Err(LookupRequestError::TooManyAttempts { cannot_process }) } else { Ok(()) @@ -141,15 +138,9 @@ pub trait RequestState { /// Get the next peer to request. Draws from the set of peers we think should have both the /// block and blob first. If that fails, we draw from the set of peers that may have either. fn get_peer(&mut self) -> Result { - let request_state = self.get_state_mut(); - let peer_id = request_state - .available_peers - .iter() - .choose(&mut rand::thread_rng()) - .copied() - .ok_or(LookupRequestError::NoPeers)?; - request_state.used_peers.insert(peer_id); - Ok(peer_id) + self.get_state_mut() + .use_rand_available_peer() + .ok_or(LookupRequestError::NoPeers) } /// Initialize `Self::RequestType`. @@ -169,29 +160,35 @@ pub trait RequestState { fn verify_response( &mut self, expected_block_root: Hash256, + peer_id: PeerId, response: Option, ) -> Result, LookupVerifyError> { - let request_state = self.get_state_mut(); - match request_state.state { - State::AwaitingDownload => { - request_state.register_failure_downloading(); - Err(LookupVerifyError::ExtraBlocksReturned) - } - State::Downloading { peer_id } => { - self.verify_response_inner(expected_block_root, response, peer_id) + let result = match *self.get_state().get_state() { + State::AwaitingDownload => Err(LookupVerifyError::ExtraBlocksReturned), + State::Downloading { peer_id: _ } => { + // TODO: We requested a download from Downloading { peer_id }, but the network + // injects a response from a different peer_id. What should we do? The peer_id to + // track for scoring is the one that actually sent the response, not the state's + self.verify_response_inner(expected_block_root, response) } - State::Processing { peer_id: _ } => match response { - Some(_) => { - // We sent the block for processing and received an extra block. - request_state.register_failure_downloading(); - Err(LookupVerifyError::ExtraBlocksReturned) - } - None => { - // This is simply the stream termination and we are already processing the - // block - Ok(None) - } + State::Processing { .. } | State::Processed { .. } => match response { + // We sent the block for processing and received an extra block. + Some(_) => Err(LookupVerifyError::ExtraBlocksReturned), + // This is simply the stream termination and we are already processing the block + None => Ok(None), }, + }; + + match result { + Ok(Some(response)) => { + self.get_state_mut().on_download_success(peer_id); + Ok(Some(response)) + } + Ok(None) => Ok(None), + Err(e) => { + self.get_state_mut().on_download_failure(); + Err(e) + } } } @@ -200,7 +197,6 @@ pub trait RequestState { &mut self, expected_block_root: Hash256, response: Option, - peer_id: PeerId, ) -> Result, LookupVerifyError>; /// A getter for the parent root of the response. Returns an `Option` because we won't know @@ -232,7 +228,7 @@ pub trait RequestState { /// Register a failure to process the block or blob. fn register_failure_downloading(&mut self) { - self.get_state_mut().register_failure_downloading() + self.get_state_mut().on_download_failure() } /* Utility methods */ @@ -274,7 +270,6 @@ impl RequestState for BlockRequestState &mut self, expected_block_root: Hash256, response: Option, - peer_id: PeerId, ) -> Result>>, LookupVerifyError> { match response { Some(block) => { @@ -285,18 +280,13 @@ impl RequestState for BlockRequestState // return an error and drop the block // NOTE: we take this is as a download failure to prevent counting the // attempt as a chain failure, but simply a peer failure. - self.state.register_failure_downloading(); Err(LookupVerifyError::RootMismatch) } else { // Return the block for processing. - self.state.state = State::Processing { peer_id }; Ok(Some(block)) } } - None => { - self.state.register_failure_downloading(); - Err(LookupVerifyError::NoBlockReturned) - } + None => Err(LookupVerifyError::NoBlockReturned), } } @@ -374,25 +364,20 @@ impl RequestState for BlobRequestState, - peer_id: PeerId, ) -> Result>, LookupVerifyError> { match blob { Some(blob) => { let received_id = blob.id(); if !self.requested_ids.contains(&received_id) { - Err(LookupVerifyError::UnrequestedBlobId(received_id)) - } else if !blob.verify_blob_sidecar_inclusion_proof().unwrap_or(false) { - Err(LookupVerifyError::InvalidInclusionProof) - } else if blob.block_root() != expected_block_root { - Err(LookupVerifyError::UnrequestedHeader) - } else { - Ok(()) + return Err(LookupVerifyError::UnrequestedBlobId(received_id)); + } + if !blob.verify_blob_sidecar_inclusion_proof().unwrap_or(false) { + return Err(LookupVerifyError::InvalidInclusionProof); + } + if blob.block_root() != expected_block_root { + return Err(LookupVerifyError::UnrequestedHeader); } - .map_err(|e| { - self.state.register_failure_downloading(); - e - })?; // State should remain downloading until we receive the stream terminator. self.requested_ids.remove(&received_id); @@ -403,7 +388,6 @@ impl RequestState for BlobRequestState { - self.state.state = State::Processing { peer_id }; let blobs = std::mem::take(&mut self.blob_download_queue); Ok(Some(blobs)) } diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index 949e1762ac7..4e1d02d38f4 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -285,12 +285,16 @@ impl BlockLookups { let mut lookup = self.single_block_lookups.remove(&id.id)?; let request_state = R::request_state_mut(&mut lookup); - if id.req_counter != request_state.get_state().req_counter { + if request_state + .get_state() + .is_current_req_counter(id.req_counter) + { + Some(lookup) + } else { // We don't want to drop the lookup, just ignore the old response. self.single_block_lookups.insert(id.id, lookup); - return None; + None } - Some(lookup) } /// Checks whether a single block lookup is waiting for a parent lookup to complete. This is @@ -374,7 +378,7 @@ impl BlockLookups { let expected_block_root = lookup.block_root(); let request_state = R::request_state_mut(&mut lookup); - match request_state.verify_response(expected_block_root, response) { + match request_state.verify_response(expected_block_root, peer_id, response) { Ok(Some(verified_response)) => { self.handle_verified_response::( seen_timestamp, @@ -415,10 +419,6 @@ impl BlockLookups { let id = lookup.id; let block_root = lookup.block_root(); - R::request_state_mut(lookup) - .get_state_mut() - .component_downloaded = true; - let cached_child = lookup.add_response::(verified_response.clone()); match cached_child { CachedChild::Ok(block) => { @@ -447,10 +447,7 @@ impl BlockLookups { // initial request. if lookup.both_components_downloaded() { lookup.penalize_blob_peer(cx); - lookup - .blob_request_state - .state - .register_failure_downloading(); + lookup.blob_request_state.state.on_download_failure(); } lookup.request_block_and_blobs(cx)?; } @@ -493,13 +490,13 @@ impl BlockLookups { if R::request_state_mut(&mut parent_lookup.current_parent_request) .get_state() - .req_counter - != id.req_counter + .is_current_req_counter(id.req_counter) { + Some(parent_lookup) + } else { self.parent_lookups.push(parent_lookup); - return None; + None } - Some(parent_lookup) } /// Process a response received from a parent lookup request. @@ -559,7 +556,7 @@ impl BlockLookups { cx: &SyncNetworkContext, parent_lookup: &mut ParentLookup, ) -> Result<(), RequestError> { - match parent_lookup.verify_response::(response, &mut self.failed_chains) { + match parent_lookup.verify_response::(peer_id, response, &mut self.failed_chains) { Ok(Some(verified_response)) => { self.handle_verified_response::( seen_timestamp, @@ -640,7 +637,7 @@ impl BlockLookups { RequestError::ChainTooLong => { self.failed_chains.insert(parent_lookup.chain_hash()); // This indicates faulty peers. - for &peer_id in parent_lookup.used_peers() { + for &peer_id in parent_lookup.all_used_peers() { cx.report_peer(peer_id, PeerAction::LowToleranceError, e.as_static()) } } @@ -653,7 +650,7 @@ impl BlockLookups { self.failed_chains.insert(parent_lookup.chain_hash()); } // This indicates faulty peers. - for &peer_id in parent_lookup.used_peers() { + for &peer_id in parent_lookup.all_used_peers() { cx.report_peer(peer_id, PeerAction::LowToleranceError, e.as_static()) } } @@ -661,6 +658,9 @@ impl BlockLookups { // This happens if the peer disconnects while the block is being // processed. Drop the request without extra penalty } + RequestError::BadState(_) => { + // Should never happen + } } } @@ -668,9 +668,12 @@ impl BlockLookups { pub fn peer_disconnected(&mut self, peer_id: &PeerId, cx: &mut SyncNetworkContext) { /* Check disconnection for single lookups */ - self.single_block_lookups.retain(|_, req| { + self.single_block_lookups.retain(|id, req| { let should_drop_lookup = req.should_drop_lookup_on_disconnected_peer(peer_id, cx, &self.log); + if should_drop_lookup { + debug!(self.log, "Dropping lookup after peer disconnected"; "id" => id, "block_root" => %req.block_root()); + } !should_drop_lookup }); @@ -844,15 +847,15 @@ impl BlockLookups { ) -> Result<(), LookupRequestError> { let request_state = R::request_state_mut(lookup); - request_state.get_state_mut().component_processed = true; + request_state + .get_state_mut() + .on_processing_success() + .map_err(LookupRequestError::BadState)?; if lookup.both_components_processed() { lookup.penalize_blob_peer(cx); // Try it again if possible. - lookup - .blob_request_state - .state - .register_failure_processing(); + lookup.blob_request_state.state.on_processing_failure(); lookup.request_block_and_blobs(cx)?; } Ok(()) @@ -900,14 +903,8 @@ impl BlockLookups { BlockError::AvailabilityCheck(e) => match e.category() { AvailabilityCheckErrorCategory::Internal => { warn!(self.log, "Internal availability check failure"; "root" => %root, "peer_id" => %peer_id, "error" => ?e); - lookup - .block_request_state - .state - .register_failure_downloading(); - lookup - .blob_request_state - .state - .register_failure_downloading(); + lookup.block_request_state.state.on_download_failure(); + lookup.blob_request_state.state.on_download_failure(); lookup.request_block_and_blobs(cx)? } AvailabilityCheckErrorCategory::Malicious => { @@ -926,10 +923,7 @@ impl BlockLookups { ); // Try it again if possible. - lookup - .block_request_state - .state - .register_failure_processing(); + lookup.block_request_state.state.on_processing_failure(); lookup.request_block_and_blobs(cx)? } } @@ -999,7 +993,7 @@ impl BlockLookups { .current_parent_request .blob_request_state .state - .register_failure_processing(); + .on_processing_failure(); match parent_lookup .current_parent_request .request_block_and_blobs(cx) @@ -1255,8 +1249,8 @@ impl BlockLookups { penalty, } => { self.failed_chains.insert(chain_hash); - for peer_source in request.all_peers() { - cx.report_peer(peer_source, penalty, "parent_chain_failure") + for peer_source in request.all_used_peers() { + cx.report_peer(*peer_source, penalty, "parent_chain_failure") } } BatchProcessResult::NonFaultyFailure => { diff --git a/beacon_node/network/src/sync/block_lookups/parent_lookup.rs b/beacon_node/network/src/sync/block_lookups/parent_lookup.rs index a3cdfd7b00b..55dd26b661e 100644 --- a/beacon_node/network/src/sync/block_lookups/parent_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/parent_lookup.rs @@ -7,7 +7,6 @@ use beacon_chain::block_verification_types::AsBlock; use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::data_availability_checker::{ChildComponents, DataAvailabilityChecker}; use beacon_chain::BeaconChainTypes; -use itertools::Itertools; use std::collections::VecDeque; use std::sync::Arc; use store::Hash256; @@ -56,6 +55,7 @@ pub enum RequestError { cannot_process: bool, }, NoPeers, + BadState(String), } impl ParentLookup { @@ -175,11 +175,11 @@ impl ParentLookup { self.current_parent_request .block_request_state .state - .register_failure_processing(); + .on_processing_failure(); self.current_parent_request .blob_request_state .state - .register_failure_processing(); + .on_processing_failure(); if let Some(components) = self.current_parent_request.child_components.as_mut() { components.downloaded_block = None; components.downloaded_blobs = <_>::default(); @@ -190,12 +190,14 @@ impl ParentLookup { /// the processing result of the block. pub fn verify_response>( &mut self, + peer_id: PeerId, block: Option, failed_chains: &mut lru_cache::LRUTimeCache, ) -> Result, ParentVerifyError> { let expected_block_root = self.current_parent_request.block_root(); let request_state = R::request_state_mut(&mut self.current_parent_request); - let root_and_verified = request_state.verify_response(expected_block_root, block)?; + let root_and_verified = + request_state.verify_response(expected_block_root, peer_id, block)?; // check if the parent of this block isn't in the failed cache. If it is, this chain should // be dropped and the peer downscored. @@ -221,20 +223,8 @@ impl ParentLookup { self.current_parent_request.add_peers(peers) } - pub fn used_peers(&self) -> impl Iterator + '_ { - self.current_parent_request - .block_request_state - .state - .used_peers - .iter() - .chain( - self.current_parent_request - .blob_request_state - .state - .used_peers - .iter(), - ) - .unique() + pub fn all_used_peers(&self) -> impl Iterator + '_ { + self.current_parent_request.all_used_peers() } } @@ -264,6 +254,7 @@ impl From for RequestError { } E::NoPeers => RequestError::NoPeers, E::SendFailed(msg) => RequestError::SendFailed(msg), + E::BadState(msg) => RequestError::BadState(msg), } } } @@ -291,6 +282,7 @@ impl RequestError { } RequestError::TooManyAttempts { cannot_process: _ } => "too_many_downloading_attempts", RequestError::NoPeers => "no_peers", + RequestError::BadState(_) => "bad_state", } } } diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index a312f6e970a..15d10c77c24 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -8,7 +8,9 @@ use beacon_chain::data_availability_checker::{ AvailabilityCheckError, DataAvailabilityChecker, MissingBlobs, }; use beacon_chain::BeaconChainTypes; +use itertools::Itertools; use lighthouse_network::PeerAction; +use rand::seq::IteratorRandom; use slog::{trace, Logger}; use std::collections::HashSet; use std::fmt::Debug; @@ -19,13 +21,6 @@ use strum::IntoStaticStr; use types::blob_sidecar::{BlobIdentifier, FixedBlobSidecarList}; use types::EthSpec; -#[derive(Debug, PartialEq, Eq)] -pub enum State { - AwaitingDownload, - Downloading { peer_id: PeerId }, - Processing { peer_id: PeerId }, -} - #[derive(Debug, PartialEq, Eq, IntoStaticStr)] pub enum LookupVerifyError { RootMismatch, @@ -48,6 +43,7 @@ pub enum LookupRequestError { }, NoPeers, SendFailed(&'static str), + BadState(String), } pub struct SingleBlockLookup { @@ -94,18 +90,16 @@ impl SingleBlockLookup { self.block_request_state.requested_block_root = block_root; self.block_request_state.state.state = State::AwaitingDownload; self.blob_request_state.state.state = State::AwaitingDownload; - self.block_request_state.state.component_downloaded = false; - self.blob_request_state.state.component_downloaded = false; - self.block_request_state.state.component_processed = false; - self.blob_request_state.state.component_processed = false; self.child_components = Some(ChildComponents::empty(block_root)); } - /// Get all unique peers across block and blob requests. - pub fn all_peers(&self) -> HashSet { - let mut all_peers = self.block_request_state.state.used_peers.clone(); - all_peers.extend(self.blob_request_state.state.used_peers.clone()); - all_peers + /// Get all unique used peers across block and blob requests. + pub fn all_used_peers(&self) -> impl Iterator + '_ { + self.block_request_state + .state + .get_used_peers() + .chain(self.blob_request_state.state.get_used_peers()) + .unique() } /// Send the necessary requests for blocks and/or blobs. This will check whether we have @@ -206,14 +200,14 @@ impl SingleBlockLookup { /// Returns true if the block has already been downloaded. pub fn both_components_downloaded(&self) -> bool { - self.block_request_state.state.component_downloaded - && self.blob_request_state.state.component_downloaded + self.block_request_state.state.is_downloaded() + && self.blob_request_state.state.is_downloaded() } /// Returns true if the block has already been downloaded. pub fn both_components_processed(&self) -> bool { - self.block_request_state.state.component_processed - && self.blob_request_state.state.component_processed + self.block_request_state.state.is_processed() + && self.blob_request_state.state.is_processed() } /// Checks both the block and blob request states to see if the peer is disconnected. @@ -304,7 +298,7 @@ impl SingleBlockLookup { if let Some(cached_child) = self.child_components.as_mut() { cached_child.clear_blobs(); } - self.blob_request_state.state.register_failure_downloading() + self.blob_request_state.state.on_download_failure() } /// This failure occurs after processing, so register a failure processing, penalize the peer @@ -314,7 +308,7 @@ impl SingleBlockLookup { if let Some(cached_child) = self.child_components.as_mut() { cached_child.clear_blobs(); } - self.blob_request_state.state.register_failure_processing() + self.blob_request_state.state.on_processing_failure() } } @@ -375,29 +369,34 @@ pub enum CachedChild { /// There was an error during consistency checks between block and blobs. Err(AvailabilityCheckError), } + +#[derive(Debug, PartialEq, Eq)] +pub enum State { + AwaitingDownload, + Downloading { peer_id: PeerId }, + Processing { peer_id: PeerId }, + Processed { peer_id: PeerId }, +} + /// Object representing the state of a single block or blob lookup request. #[derive(PartialEq, Eq, Debug)] pub struct SingleLookupRequestState { /// State of this request. - pub state: State, + state: State, /// Peers that should have this block or blob. - pub available_peers: HashSet, + available_peers: HashSet, /// Peers from which we have requested this block. - pub used_peers: HashSet, + used_peers: HashSet, /// How many times have we attempted to process this block or blob. - pub failed_processing: u8, + failed_processing: u8, /// How many times have we attempted to download this block or blob. - pub failed_downloading: u8, - /// Whether or not we have downloaded this block or blob. - pub component_downloaded: bool, - /// Whether or not we have processed this block or blob. - pub component_processed: bool, + failed_downloading: u8, /// Should be incremented everytime this request is retried. The purpose of this is to /// differentiate retries of the same block/blob request within a lookup. We currently penalize /// peers and retry requests prior to receiving the stream terminator. This means responses /// from a prior request may arrive after a new request has been sent, this counter allows /// us to differentiate these two responses. - pub req_counter: u32, + req_counter: u32, } impl SingleLookupRequestState { @@ -413,30 +412,83 @@ impl SingleLookupRequestState { used_peers: HashSet::default(), failed_processing: 0, failed_downloading: 0, - component_downloaded: false, - component_processed: false, req_counter: 0, } } - /// Registers a failure in processing a block. - pub fn register_failure_processing(&mut self) { - self.failed_processing = self.failed_processing.saturating_add(1); - self.state = State::AwaitingDownload; + // TODO: Should not leak the enum state + pub fn get_state(&self) -> &State { + &self.state + } + + pub fn is_current_req_counter(&self, req_counter: u32) -> bool { + self.req_counter == req_counter + } + + pub fn is_awaiting_download(&self) -> bool { + matches!(self.state, State::AwaitingDownload) + } + + pub fn is_downloaded(&self) -> bool { + match self.state { + State::AwaitingDownload => false, + State::Downloading { .. } => false, + State::Processing { .. } => true, + State::Processed { .. } => true, + } + } + + pub fn is_processed(&self) -> bool { + match self.state { + State::AwaitingDownload => false, + State::Downloading { .. } => false, + State::Processing { .. } => false, + State::Processed { .. } => true, + } + } + + pub fn on_download_start(&mut self, peer_id: PeerId) -> u32 { + self.state = State::Downloading { peer_id }; + self.req_counter += 1; + self.req_counter } /// Registers a failure in downloading a block. This might be a peer disconnection or a wrong /// block. - pub fn register_failure_downloading(&mut self) { + pub fn on_download_failure(&mut self) { self.failed_downloading = self.failed_downloading.saturating_add(1); self.state = State::AwaitingDownload; } + pub fn on_download_success(&mut self, peer_id: PeerId) { + self.state = State::Processing { peer_id }; + } + + /// Registers a failure in processing a block. + pub fn on_processing_failure(&mut self) { + self.failed_processing = self.failed_processing.saturating_add(1); + self.state = State::AwaitingDownload; + } + + pub fn on_processing_success(&mut self) -> Result<(), String> { + match &self.state { + State::Processing { peer_id } => { + self.state = State::Processed { peer_id: *peer_id }; + Ok(()) + } + other => Err(format!("not in processing state: {}", other).to_string()), + } + } + /// The total number of failures, whether it be processing or downloading. pub fn failed_attempts(&self) -> u8 { self.failed_processing + self.failed_downloading } + pub fn more_failed_processing_attempts(&self) -> bool { + self.failed_processing >= self.failed_downloading + } + /// This method should be used for peers wrapped in `PeerId::BlockAndBlobs`. pub fn add_peer(&mut self, peer_id: &PeerId) { self.available_peers.insert(*peer_id); @@ -448,7 +500,7 @@ impl SingleLookupRequestState { if let State::Downloading { peer_id } = &self.state { if peer_id == dc_peer_id { // Peer disconnected before providing a block - self.register_failure_downloading(); + self.on_download_failure(); return Err(()); } } @@ -459,10 +511,25 @@ impl SingleLookupRequestState { /// returns an error. pub fn processing_peer(&self) -> Result { match &self.state { - State::Processing { peer_id } => Ok(*peer_id), + State::Processing { peer_id } | State::Processed { peer_id } => Ok(*peer_id), other => Err(format!("not in processing state: {}", other).to_string()), } } + + pub fn get_used_peers(&self) -> impl Iterator { + self.used_peers.iter() + } + + /// Selects a random peer from available peers if any, inserts it in used peers and returns it. + pub fn use_rand_available_peer(&mut self) -> Option { + let peer_id = self + .available_peers + .iter() + .choose(&mut rand::thread_rng()) + .copied()?; + self.used_peers.insert(peer_id); + Some(peer_id) + } } impl slog::Value for SingleBlockLookup { @@ -509,6 +576,7 @@ impl slog::Value for SingleLookupRequestState { State::Processing { peer_id } => { serializer.emit_arguments("processing_peer", &format_args!("{}", peer_id))? } + State::Processed { .. } => "processed".serialize(record, "state", serializer)?, } serializer.emit_u8("failed_downloads", self.failed_downloading)?; serializer.emit_u8("failed_processing", self.failed_processing)?; @@ -522,6 +590,7 @@ impl std::fmt::Display for State { State::AwaitingDownload => write!(f, "AwaitingDownload"), State::Downloading { .. } => write!(f, "Downloading"), State::Processing { .. } => write!(f, "Processing"), + State::Processed { .. } => write!(f, "Processed"), } } } @@ -608,6 +677,7 @@ mod tests { as RequestState>::verify_response( &mut sl.block_request_state, block.canonical_root(), + peer_id, Some(block.into()), ) .unwrap() @@ -647,7 +717,7 @@ mod tests { &spec, ) .unwrap(); - sl.block_request_state.state.register_failure_downloading(); + sl.block_request_state.state.on_download_failure(); } // Now we receive the block and send it for processing @@ -661,13 +731,14 @@ mod tests { as RequestState>::verify_response( &mut sl.block_request_state, block.canonical_root(), + peer_id, Some(block.into()), ) .unwrap() .unwrap(); // One processing failure maxes the available attempts - sl.block_request_state.state.register_failure_processing(); + sl.block_request_state.state.on_processing_failure(); assert_eq!( as RequestState>::build_request( &mut sl.block_request_state, diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 15595ecde5c..a868a092d3d 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -637,6 +637,7 @@ impl SyncManager { self.handle_unknown_block_root(peer_id, block_root); } SyncMessage::Disconnect(peer_id) => { + debug!(self.log, "Received disconnected message"; "peer_id" => %peer_id); self.peer_disconnect(&peer_id); } SyncMessage::RpcError { From f5e0404fb8c5ad924b5d0f0f4e953f7a3778e327 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 16 Apr 2024 06:20:50 +0900 Subject: [PATCH 06/37] Ensure proper ReqResp blocks_by_* response stream termination (#5582) * Ensure proper ReqResp blocks_by_* response stream termination * retrigger CI --- .../network_beacon_processor/rpc_methods.rs | 119 ++++++++++-------- 1 file changed, 66 insertions(+), 53 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs index 81c8f662ee9..fb813723920 100644 --- a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs @@ -134,13 +134,37 @@ impl NetworkBeaconProcessor { request_id: PeerRequestId, request: BlocksByRootRequest, ) { + self.terminate_response_stream( + peer_id, + request_id, + self.clone() + .handle_blocks_by_root_request_inner(executor, peer_id, request_id, request) + .await, + Response::BlocksByRoot, + ); + } + + /// Handle a `BlocksByRoot` request from the peer. + pub async fn handle_blocks_by_root_request_inner( + self: Arc, + executor: TaskExecutor, + peer_id: PeerId, + request_id: PeerRequestId, + request: BlocksByRootRequest, + ) -> Result<(), (RPCResponseErrorCode, &'static str)> { let requested_blocks = request.block_roots().len(); let mut block_stream = match self .chain .get_blocks_checking_caches(request.block_roots().to_vec(), &executor) { Ok(block_stream) => block_stream, - Err(e) => return error!(self.log, "Error getting block stream"; "error" => ?e), + Err(e) => { + error!(self.log, "Error getting block stream"; "error" => ?e); + return Err(( + RPCResponseErrorCode::ServerError, + "Error getting block stream", + )); + } }; // Fetching blocks is async because it may have to hit the execution layer for payloads. let mut send_block_count = 0; @@ -169,13 +193,10 @@ impl NetworkBeaconProcessor { "block_root" => ?root, "reason" => "execution layer not synced", ); - // send the stream terminator - return self.send_error_response( - peer_id, + return Err(( RPCResponseErrorCode::ResourceUnavailable, - "Execution layer not synced".into(), - request_id, - ); + "Execution layer not synced", + )); } Err(e) => { debug!( @@ -196,8 +217,7 @@ impl NetworkBeaconProcessor { "returned" => %send_block_count ); - // send stream termination - self.send_response(peer_id, Response::BlocksByRoot(None), request_id); + Ok(()) } /// Handle a `BlobsByRoot` request from the peer. @@ -380,6 +400,24 @@ impl NetworkBeaconProcessor { request_id: PeerRequestId, req: BlocksByRangeRequest, ) { + self.terminate_response_stream( + peer_id, + request_id, + self.clone() + .handle_blocks_by_range_request_inner(executor, peer_id, request_id, req) + .await, + Response::BlocksByRange, + ); + } + + /// Handle a `BlocksByRange` request from the peer. + pub async fn handle_blocks_by_range_request_inner( + self: Arc, + executor: TaskExecutor, + peer_id: PeerId, + request_id: PeerRequestId, + req: BlocksByRangeRequest, + ) -> Result<(), (RPCResponseErrorCode, &'static str)> { debug!(self.log, "Received BlocksByRange Request"; "peer_id" => %peer_id, "count" => req.count(), @@ -401,12 +439,10 @@ impl NetworkBeaconProcessor { } }); if *req.count() > max_request_size { - return self.send_error_response( - peer_id, + return Err(( RPCResponseErrorCode::InvalidRequest, - format!("Request exceeded max size {max_request_size}"), - request_id, - ); + "Request exceeded max size", + )); } let forwards_block_root_iter = match self @@ -424,25 +460,15 @@ impl NetworkBeaconProcessor { "requested_slot" => slot, "oldest_known_slot" => oldest_block_slot ); - return self.send_error_response( - peer_id, - RPCResponseErrorCode::ResourceUnavailable, - "Backfilling".into(), - request_id, - ); + return Err((RPCResponseErrorCode::ResourceUnavailable, "Backfilling")); } Err(e) => { - self.send_error_response( - peer_id, - RPCResponseErrorCode::ServerError, - "Database error".into(), - request_id, - ); - return error!(self.log, "Unable to obtain root iter"; + error!(self.log, "Unable to obtain root iter"; "request" => ?req, "peer" => %peer_id, "error" => ?e ); + return Err((RPCResponseErrorCode::ServerError, "Database error")); } }; @@ -468,11 +494,12 @@ impl NetworkBeaconProcessor { let block_roots = match maybe_block_roots { Ok(block_roots) => block_roots, Err(e) => { - return error!(self.log, "Error during iteration over blocks"; + error!(self.log, "Error during iteration over blocks"; "request" => ?req, "peer" => %peer_id, "error" => ?e - ) + ); + return Err((RPCResponseErrorCode::ServerError, "Iteration error")); } }; @@ -481,7 +508,10 @@ impl NetworkBeaconProcessor { let mut block_stream = match self.chain.get_blocks(block_roots, &executor) { Ok(block_stream) => block_stream, - Err(e) => return error!(self.log, "Error getting block stream"; "error" => ?e), + Err(e) => { + error!(self.log, "Error getting block stream"; "error" => ?e); + return Err((RPCResponseErrorCode::ServerError, "Iterator error")); + } }; // Fetching blocks is async because it may have to hit the execution layer for payloads. @@ -511,12 +541,7 @@ impl NetworkBeaconProcessor { "peer" => %peer_id, "request_root" => ?root ); - return self.send_error_response( - peer_id, - RPCResponseErrorCode::ServerError, - "Database inconsistency".into(), - request_id, - ); + return Err((RPCResponseErrorCode::ServerError, "Database inconsistency")); } Err(BeaconChainError::BlockHashMissingFromExecutionLayer(_)) => { debug!( @@ -526,12 +551,10 @@ impl NetworkBeaconProcessor { "reason" => "execution layer not synced", ); // send the stream terminator - return self.send_error_response( - peer_id, + return Err(( RPCResponseErrorCode::ResourceUnavailable, - "Execution layer not synced".into(), - request_id, - ); + "Execution layer not synced", + )); } Err(e) => { if matches!( @@ -556,12 +579,7 @@ impl NetworkBeaconProcessor { } // send the stream terminator - return self.send_error_response( - peer_id, - RPCResponseErrorCode::ServerError, - "Failed fetching blocks".into(), - request_id, - ); + return Err((RPCResponseErrorCode::ServerError, "Failed fetching blocks")); } } } @@ -594,12 +612,7 @@ impl NetworkBeaconProcessor { ); } - // send the stream terminator - self.send_network_message(NetworkMessage::SendResponse { - peer_id, - response: Response::BlocksByRange(None), - id: request_id, - }); + Ok(()) } /// Handle a `BlobsByRange` request from the peer. From e5b8d1237d9eb65d56c53fd827e049b1704bbc87 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 16 Apr 2024 21:59:53 +0900 Subject: [PATCH 07/37] Use Action in single_block_component_processed (#5564) * Use Action in single_block_component_processed * Merge branch 'unstable' of https://github.com/sigp/lighthouse into single_block_component_processed-action * fix compile after merge * add continue action for when we are awaiting other parts of missing components --- .../network/src/sync/block_lookups/mod.rs | 242 +++++++++--------- 1 file changed, 126 insertions(+), 116 deletions(-) diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index 4e1d02d38f4..a5826bcb3d8 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -45,6 +45,13 @@ pub type DownloadedBlock = (Hash256, RpcBlock); const FAILED_CHAINS_CACHE_EXPIRY_SECONDS: u64 = 60; pub const SINGLE_BLOCK_LOOKUP_MAX_ATTEMPTS: u8 = 3; +enum Action { + Retry, + ParentUnknown { parent_root: Hash256, slot: Slot }, + Drop, + Continue, +} + pub struct BlockLookups { /// Parent chain lookups being downloaded. parent_lookups: SmallVec<[ParentLookup; 3]>, @@ -773,7 +780,7 @@ impl BlockLookups { return; }; - let root = lookup.block_root(); + let block_root = lookup.block_root(); let request_state = R::request_state_mut(&mut lookup); let peer_id = match request_state.get_state().processing_peer() { @@ -787,28 +794,49 @@ impl BlockLookups { self.log, "Block component processed for lookup"; "response_type" => ?R::response_type(), - "block_root" => ?root, + "block_root" => ?block_root, "result" => ?result, "id" => target_id, ); - match result { - BlockProcessingResult::Ok(status) => match status { - AvailabilityProcessingStatus::Imported(root) => { - trace!(self.log, "Single block processing succeeded"; "block" => %root); - } - AvailabilityProcessingStatus::MissingComponents(_, _block_root) => { - match self.handle_missing_components::(cx, &mut lookup) { - Ok(()) => { - self.single_block_lookups.insert(target_id, lookup); - } - Err(e) => { - // Drop with an additional error. - warn!(self.log, "Single block lookup failed"; "block" => %root, "error" => ?e); - } - } + let action = match result { + BlockProcessingResult::Ok(AvailabilityProcessingStatus::Imported(_)) + | BlockProcessingResult::Err(BlockError::BlockIsAlreadyKnown { .. }) => { + // Successfully imported + trace!(self.log, "Single block processing succeeded"; "block" => %block_root); + Action::Drop + } + + BlockProcessingResult::Ok(AvailabilityProcessingStatus::MissingComponents( + _, + _block_root, + )) => { + // `on_processing_success` is called here to ensure the request state is updated prior to checking + // if both components have been processed. + if R::request_state_mut(&mut lookup) + .get_state_mut() + .on_processing_success() + .is_err() + { + warn!( + self.log, + "Single block processing state incorrect"; + "action" => "dropping single block request" + ); + Action::Drop + // If this was the result of a block request, we can't determined if the block peer did anything + // wrong. If we already had both a block and blobs response processed, we should penalize the + // blobs peer because they did not provide all blobs on the initial request. + } else if lookup.both_components_processed() { + lookup.penalize_blob_peer(cx); + + // Try it again if possible. + lookup.blob_request_state.state.on_processing_failure(); + Action::Retry + } else { + Action::Continue } - }, + } BlockProcessingResult::Ignored => { // Beacon processor signalled to ignore the block processing result. // This implies that the cpu is overloaded. Drop the request. @@ -817,118 +845,88 @@ impl BlockLookups { "Single block processing was ignored, cpu might be overloaded"; "action" => "dropping single block request" ); + Action::Drop } BlockProcessingResult::Err(e) => { - match self.handle_single_lookup_block_error(cx, lookup, peer_id, e) { - Ok(Some(lookup)) => { - self.single_block_lookups.insert(target_id, lookup); + let root = lookup.block_root(); + trace!(self.log, "Single block processing failed"; "block" => %root, "error" => %e); + match e { + BlockError::BeaconChainError(e) => { + // Internal error + error!(self.log, "Beacon chain error processing single block"; "block_root" => %root, "error" => ?e); + Action::Drop } - Ok(None) => { - // Drop without an additional error. + BlockError::ParentUnknown(block) => { + let slot = block.slot(); + let parent_root = block.parent_root(); + lookup.add_child_components(block.into()); + Action::ParentUnknown { parent_root, slot } } - Err(e) => { - // Drop with an additional error. - warn!(self.log, "Single block lookup failed"; "block" => %root, "error" => ?e); + ref e @ BlockError::ExecutionPayloadError(ref epe) if !epe.penalize_peer() => { + // These errors indicate that the execution layer is offline + // and failed to validate the execution payload. Do not downscore peer. + debug!( + self.log, + "Single block lookup failed. Execution layer is offline / unsynced / misconfigured"; + "root" => %root, + "error" => ?e + ); + Action::Drop + } + BlockError::AvailabilityCheck(e) => match e.category() { + AvailabilityCheckErrorCategory::Internal => { + warn!(self.log, "Internal availability check failure"; "root" => %root, "peer_id" => %peer_id, "error" => ?e); + lookup.block_request_state.state.on_download_failure(); + lookup.blob_request_state.state.on_download_failure(); + Action::Retry + } + AvailabilityCheckErrorCategory::Malicious => { + warn!(self.log, "Availability check failure"; "root" => %root, "peer_id" => %peer_id, "error" => ?e); + lookup.handle_availability_check_failure(cx); + Action::Retry + } + }, + other => { + warn!(self.log, "Peer sent invalid block in single block lookup"; "root" => %root, "error" => ?other, "peer_id" => %peer_id); + if let Ok(block_peer) = lookup.block_request_state.state.processing_peer() { + cx.report_peer( + block_peer, + PeerAction::MidToleranceError, + "single_block_failure", + ); + + lookup.block_request_state.state.on_processing_failure(); + } + Action::Retry } } } }; - } - - /// Handles a `MissingComponents` block processing error. Handles peer scoring and retries. - /// - /// If this was the result of a block request, we can't determined if the block peer did anything - /// wrong. If we already had both a block and blobs response processed, we should penalize the - /// blobs peer because they did not provide all blobs on the initial request. - fn handle_missing_components>( - &self, - cx: &SyncNetworkContext, - lookup: &mut SingleBlockLookup, - ) -> Result<(), LookupRequestError> { - let request_state = R::request_state_mut(lookup); - - request_state - .get_state_mut() - .on_processing_success() - .map_err(LookupRequestError::BadState)?; - if lookup.both_components_processed() { - lookup.penalize_blob_peer(cx); - - // Try it again if possible. - lookup.blob_request_state.state.on_processing_failure(); - lookup.request_block_and_blobs(cx)?; - } - Ok(()) - } - /// Handles peer scoring and retries related to a `BlockError` in response to a single block - /// or blob lookup processing result. - fn handle_single_lookup_block_error( - &mut self, - cx: &mut SyncNetworkContext, - mut lookup: SingleBlockLookup, - peer_id: PeerId, - e: BlockError, - ) -> Result>, LookupRequestError> { - let root = lookup.block_root(); - trace!(self.log, "Single block processing failed"; "block" => %root, "error" => %e); - match e { - BlockError::BlockIsAlreadyKnown(_) => { - // No error here - return Ok(None); - } - BlockError::BeaconChainError(e) => { - // Internal error - error!(self.log, "Beacon chain error processing single block"; "block_root" => %root, "error" => ?e); - return Ok(None); + match action { + Action::Retry => { + if let Err(e) = lookup.request_block_and_blobs(cx) { + warn!(self.log, "Single block lookup failed"; "block_root" => %block_root, "error" => ?e); + // Failed with too many retries, drop with noop + self.update_metrics(); + } else { + self.single_block_lookups.insert(target_id, lookup); + } } - BlockError::ParentUnknown(block) => { - let slot = block.slot(); - let parent_root = block.parent_root(); - lookup.add_child_components(block.into()); - lookup.request_block_and_blobs(cx)?; - self.search_parent(slot, root, parent_root, peer_id, cx); + Action::ParentUnknown { parent_root, slot } => { + // TODO: Consider including all peers from the lookup, claiming to know this block, not + // just the one that sent this specific block + self.search_parent(slot, block_root, parent_root, peer_id, cx); + self.single_block_lookups.insert(target_id, lookup); } - ref e @ BlockError::ExecutionPayloadError(ref epe) if !epe.penalize_peer() => { - // These errors indicate that the execution layer is offline - // and failed to validate the execution payload. Do not downscore peer. - debug!( - self.log, - "Single block lookup failed. Execution layer is offline / unsynced / misconfigured"; - "root" => %root, - "error" => ?e - ); - return Ok(None); + Action::Drop => { + // drop with noop + self.update_metrics(); } - BlockError::AvailabilityCheck(e) => match e.category() { - AvailabilityCheckErrorCategory::Internal => { - warn!(self.log, "Internal availability check failure"; "root" => %root, "peer_id" => %peer_id, "error" => ?e); - lookup.block_request_state.state.on_download_failure(); - lookup.blob_request_state.state.on_download_failure(); - lookup.request_block_and_blobs(cx)? - } - AvailabilityCheckErrorCategory::Malicious => { - warn!(self.log, "Availability check failure"; "root" => %root, "peer_id" => %peer_id, "error" => ?e); - lookup.handle_availability_check_failure(cx); - lookup.request_block_and_blobs(cx)? - } - }, - other => { - warn!(self.log, "Peer sent invalid block in single block lookup"; "root" => %root, "error" => ?other, "peer_id" => %peer_id); - if let Ok(block_peer) = lookup.block_request_state.state.processing_peer() { - cx.report_peer( - block_peer, - PeerAction::MidToleranceError, - "single_block_failure", - ); - - // Try it again if possible. - lookup.block_request_state.state.on_processing_failure(); - lookup.request_block_and_blobs(cx)? - } + Action::Continue => { + self.single_block_lookups.insert(target_id, lookup); } } - Ok(Some(lookup)) } pub fn parent_block_processed( @@ -1369,4 +1367,16 @@ impl BlockLookups { pub fn drop_parent_chain_requests(&mut self) -> usize { self.parent_lookups.drain(..).len() } + + pub fn update_metrics(&self) { + metrics::set_gauge( + &metrics::SYNC_SINGLE_BLOCK_LOOKUPS, + self.single_block_lookups.len() as i64, + ); + + metrics::set_gauge( + &metrics::SYNC_PARENT_BLOCK_LOOKUPS, + self.parent_lookups.len() as i64, + ); + } } From f68989815c9b66b33d98a593cb331c0c2b291b92 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Tue, 16 Apr 2024 09:56:00 -0500 Subject: [PATCH 08/37] Restore Log on Error & Spawn Blocking in Streamer (#5585) * Restore Logging in Error Cases * Use Spawn Blocking for Loading Blocks in Streamer * Merge remote-tracking branch 'upstream/unstable' into request_logging_spawn_blocking * Address Sean's Comments * save a clone --- .../beacon_chain/src/beacon_block_streamer.rs | 89 +++++++----- beacon_node/beacon_chain/src/beacon_chain.rs | 8 +- .../src/network_beacon_processor/mod.rs | 6 +- .../network_beacon_processor/rpc_methods.rs | 134 ++++++++++-------- 4 files changed, 131 insertions(+), 106 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_block_streamer.rs b/beacon_node/beacon_chain/src/beacon_block_streamer.rs index bbd5bfcac9a..4f413ce2a86 100644 --- a/beacon_node/beacon_chain/src/beacon_block_streamer.rs +++ b/beacon_node/beacon_chain/src/beacon_block_streamer.rs @@ -1,10 +1,9 @@ use crate::{metrics, BeaconChain, BeaconChainError, BeaconChainTypes}; use execution_layer::{ExecutionLayer, ExecutionPayloadBodyV1}; -use slog::{crit, debug, Logger}; +use slog::{crit, debug, error, Logger}; use std::collections::HashMap; use std::sync::Arc; use store::{DatabaseBlock, ExecutionPayloadDeneb}; -use task_executor::TaskExecutor; use tokio::sync::{ mpsc::{self, UnboundedSender}, RwLock, @@ -395,18 +394,18 @@ impl BeaconBlockStreamer { pub fn new( beacon_chain: &Arc>, check_caches: CheckCaches, - ) -> Result { + ) -> Result, BeaconChainError> { let execution_layer = beacon_chain .execution_layer .as_ref() .ok_or(BeaconChainError::ExecutionLayerMissing)? .clone(); - Ok(Self { + Ok(Arc::new(Self { execution_layer, check_caches, beacon_chain: beacon_chain.clone(), - }) + })) } fn check_caches(&self, root: Hash256) -> Option>> { @@ -425,30 +424,44 @@ impl BeaconBlockStreamer { } } - fn load_payloads(&self, block_roots: Vec) -> Vec<(Hash256, LoadResult)> { - let mut db_blocks = Vec::new(); - - for root in block_roots { - if let Some(cached_block) = self.check_caches(root).map(LoadedBeaconBlock::Full) { - db_blocks.push((root, Ok(Some(cached_block)))); - continue; - } - - match self.beacon_chain.store.try_get_full_block(&root) { - Err(e) => db_blocks.push((root, Err(e.into()))), - Ok(opt_block) => db_blocks.push(( - root, - Ok(opt_block.map(|db_block| match db_block { - DatabaseBlock::Full(block) => LoadedBeaconBlock::Full(Arc::new(block)), - DatabaseBlock::Blinded(block) => { - LoadedBeaconBlock::Blinded(Box::new(block)) + async fn load_payloads( + self: &Arc, + block_roots: Vec, + ) -> Result)>, BeaconChainError> { + let streamer = self.clone(); + // Loading from the DB is slow -> spawn a blocking task + self.beacon_chain + .spawn_blocking_handle( + move || { + let mut db_blocks = Vec::new(); + for root in block_roots { + if let Some(cached_block) = + streamer.check_caches(root).map(LoadedBeaconBlock::Full) + { + db_blocks.push((root, Ok(Some(cached_block)))); + continue; } - })), - )), - } - } - db_blocks + match streamer.beacon_chain.store.try_get_full_block(&root) { + Err(e) => db_blocks.push((root, Err(e.into()))), + Ok(opt_block) => db_blocks.push(( + root, + Ok(opt_block.map(|db_block| match db_block { + DatabaseBlock::Full(block) => { + LoadedBeaconBlock::Full(Arc::new(block)) + } + DatabaseBlock::Blinded(block) => { + LoadedBeaconBlock::Blinded(Box::new(block)) + } + })), + )), + } + } + db_blocks + }, + "load_beacon_blocks", + ) + .await } /// Pre-process the loaded blocks into execution engine requests. @@ -549,7 +562,7 @@ impl BeaconBlockStreamer { // used when the execution engine doesn't support the payload bodies methods async fn stream_blocks_fallback( - &self, + self: Arc, block_roots: Vec, sender: UnboundedSender<(Hash256, Arc>)>, ) { @@ -575,7 +588,7 @@ impl BeaconBlockStreamer { } async fn stream_blocks( - &self, + self: Arc, block_roots: Vec, sender: UnboundedSender<(Hash256, Arc>)>, ) { @@ -584,7 +597,17 @@ impl BeaconBlockStreamer { let mut n_sent = 0usize; let mut engine_requests = 0usize; - let payloads = self.load_payloads(block_roots); + let payloads = match self.load_payloads(block_roots).await { + Ok(payloads) => payloads, + Err(e) => { + error!( + self.beacon_chain.log, + "BeaconBlockStreamer: Failed to load payloads"; + "error" => ?e + ); + return; + } + }; let requests = self.get_requests(payloads).await; for (root, request) in requests { @@ -624,7 +647,7 @@ impl BeaconBlockStreamer { } pub async fn stream( - self, + self: Arc, block_roots: Vec, sender: UnboundedSender<(Hash256, Arc>)>, ) { @@ -650,9 +673,8 @@ impl BeaconBlockStreamer { } pub fn launch_stream( - self, + self: Arc, block_roots: Vec, - executor: &TaskExecutor, ) -> impl Stream>)> { let (block_tx, block_rx) = mpsc::unbounded_channel(); debug!( @@ -660,6 +682,7 @@ impl BeaconBlockStreamer { "Launching a BeaconBlockStreamer"; "blocks" => block_roots.len(), ); + let executor = self.beacon_chain.task_executor.clone(); executor.spawn(self.stream(block_roots, block_tx), "get_blocks_sender"); UnboundedReceiverStream::new(block_rx) } diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 7c497e74584..b3790024f81 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1139,7 +1139,6 @@ impl BeaconChain { pub fn get_blocks_checking_caches( self: &Arc, block_roots: Vec, - executor: &TaskExecutor, ) -> Result< impl Stream< Item = ( @@ -1149,14 +1148,12 @@ impl BeaconChain { >, Error, > { - Ok(BeaconBlockStreamer::::new(self, CheckCaches::Yes)? - .launch_stream(block_roots, executor)) + Ok(BeaconBlockStreamer::::new(self, CheckCaches::Yes)?.launch_stream(block_roots)) } pub fn get_blocks( self: &Arc, block_roots: Vec, - executor: &TaskExecutor, ) -> Result< impl Stream< Item = ( @@ -1166,8 +1163,7 @@ impl BeaconChain { >, Error, > { - Ok(BeaconBlockStreamer::::new(self, CheckCaches::No)? - .launch_stream(block_roots, executor)) + Ok(BeaconBlockStreamer::::new(self, CheckCaches::No)?.launch_stream(block_roots)) } pub fn get_blobs_checking_early_attester_cache( diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index 27b9e676da6..f10646c7414 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -509,9 +509,8 @@ impl NetworkBeaconProcessor { ) -> Result<(), Error> { let processor = self.clone(); let process_fn = async move { - let executor = processor.executor.clone(); processor - .handle_blocks_by_range_request(executor, peer_id, request_id, request) + .handle_blocks_by_range_request(peer_id, request_id, request) .await; }; @@ -530,9 +529,8 @@ impl NetworkBeaconProcessor { ) -> Result<(), Error> { let processor = self.clone(); let process_fn = async move { - let executor = processor.executor.clone(); processor - .handle_blocks_by_root_request(executor, peer_id, request_id, request) + .handle_blocks_by_root_request(peer_id, request_id, request) .await; }; diff --git a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs index fb813723920..1e72dc42578 100644 --- a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs @@ -11,7 +11,6 @@ use slog::{debug, error, warn}; use slot_clock::SlotClock; use std::collections::{hash_map::Entry, HashMap}; use std::sync::Arc; -use task_executor::TaskExecutor; use tokio_stream::StreamExt; use types::blob_sidecar::BlobIdentifier; use types::{Epoch, EthSpec, ForkName, Hash256, Slot}; @@ -129,7 +128,6 @@ impl NetworkBeaconProcessor { /// Handle a `BlocksByRoot` request from the peer. pub async fn handle_blocks_by_root_request( self: Arc, - executor: TaskExecutor, peer_id: PeerId, request_id: PeerRequestId, request: BlocksByRootRequest, @@ -138,7 +136,7 @@ impl NetworkBeaconProcessor { peer_id, request_id, self.clone() - .handle_blocks_by_root_request_inner(executor, peer_id, request_id, request) + .handle_blocks_by_root_request_inner(peer_id, request_id, request) .await, Response::BlocksByRoot, ); @@ -147,15 +145,24 @@ impl NetworkBeaconProcessor { /// Handle a `BlocksByRoot` request from the peer. pub async fn handle_blocks_by_root_request_inner( self: Arc, - executor: TaskExecutor, peer_id: PeerId, request_id: PeerRequestId, request: BlocksByRootRequest, ) -> Result<(), (RPCResponseErrorCode, &'static str)> { + let log_results = |peer_id, requested_blocks, send_block_count| { + debug!( + self.log, + "BlocksByRoot outgoing response processed"; + "peer" => %peer_id, + "requested" => requested_blocks, + "returned" => %send_block_count + ); + }; + let requested_blocks = request.block_roots().len(); let mut block_stream = match self .chain - .get_blocks_checking_caches(request.block_roots().to_vec(), &executor) + .get_blocks_checking_caches(request.block_roots().to_vec()) { Ok(block_stream) => block_stream, Err(e) => { @@ -193,6 +200,7 @@ impl NetworkBeaconProcessor { "block_root" => ?root, "reason" => "execution layer not synced", ); + log_results(peer_id, requested_blocks, send_block_count); return Err(( RPCResponseErrorCode::ResourceUnavailable, "Execution layer not synced", @@ -209,13 +217,7 @@ impl NetworkBeaconProcessor { } } } - debug!( - self.log, - "Received BlocksByRoot Request"; - "peer" => %peer_id, - "requested" => requested_blocks, - "returned" => %send_block_count - ); + log_results(peer_id, requested_blocks, send_block_count); Ok(()) } @@ -302,7 +304,7 @@ impl NetworkBeaconProcessor { } debug!( self.log, - "Received BlobsByRoot Request"; + "BlobsByRoot outgoing response processed"; "peer" => %peer_id, "request_root" => %requested_root, "request_indices" => ?requested_indices, @@ -395,7 +397,6 @@ impl NetworkBeaconProcessor { /// Handle a `BlocksByRange` request from the peer. pub async fn handle_blocks_by_range_request( self: Arc, - executor: TaskExecutor, peer_id: PeerId, request_id: PeerRequestId, req: BlocksByRangeRequest, @@ -404,7 +405,7 @@ impl NetworkBeaconProcessor { peer_id, request_id, self.clone() - .handle_blocks_by_range_request_inner(executor, peer_id, request_id, req) + .handle_blocks_by_range_request_inner(peer_id, request_id, req) .await, Response::BlocksByRange, ); @@ -413,7 +414,6 @@ impl NetworkBeaconProcessor { /// Handle a `BlocksByRange` request from the peer. pub async fn handle_blocks_by_range_request_inner( self: Arc, - executor: TaskExecutor, peer_id: PeerId, request_id: PeerRequestId, req: BlocksByRangeRequest, @@ -506,7 +506,37 @@ impl NetworkBeaconProcessor { // remove all skip slots let block_roots = block_roots.into_iter().flatten().collect::>(); - let mut block_stream = match self.chain.get_blocks(block_roots, &executor) { + let current_slot = self + .chain + .slot() + .unwrap_or_else(|_| self.chain.slot_clock.genesis_slot()); + + let log_results = |req: BlocksByRangeRequest, peer_id, blocks_sent| { + if blocks_sent < (*req.count() as usize) { + debug!( + self.log, + "BlocksByRange outgoing response processed"; + "peer" => %peer_id, + "msg" => "Failed to return all requested blocks", + "start_slot" => req.start_slot(), + "current_slot" => current_slot, + "requested" => req.count(), + "returned" => blocks_sent + ); + } else { + debug!( + self.log, + "BlocksByRange outgoing response processed"; + "peer" => %peer_id, + "start_slot" => req.start_slot(), + "current_slot" => current_slot, + "requested" => req.count(), + "returned" => blocks_sent + ); + } + }; + + let mut block_stream = match self.chain.get_blocks(block_roots) { Ok(block_stream) => block_stream, Err(e) => { error!(self.log, "Error getting block stream"; "error" => ?e); @@ -516,7 +546,6 @@ impl NetworkBeaconProcessor { // Fetching blocks is async because it may have to hit the execution layer for payloads. let mut blocks_sent = 0; - while let Some((root, result)) = block_stream.next().await { match result.as_ref() { Ok(Some(block)) => { @@ -541,6 +570,7 @@ impl NetworkBeaconProcessor { "peer" => %peer_id, "request_root" => ?root ); + log_results(req, peer_id, blocks_sent); return Err((RPCResponseErrorCode::ServerError, "Database inconsistency")); } Err(BeaconChainError::BlockHashMissingFromExecutionLayer(_)) => { @@ -550,6 +580,7 @@ impl NetworkBeaconProcessor { "block_root" => ?root, "reason" => "execution layer not synced", ); + log_results(req, peer_id, blocks_sent); // send the stream terminator return Err(( RPCResponseErrorCode::ResourceUnavailable, @@ -577,41 +608,14 @@ impl NetworkBeaconProcessor { "error" => ?e ); } - + log_results(req, peer_id, blocks_sent); // send the stream terminator return Err((RPCResponseErrorCode::ServerError, "Failed fetching blocks")); } } } - let current_slot = self - .chain - .slot() - .unwrap_or_else(|_| self.chain.slot_clock.genesis_slot()); - - if blocks_sent < (*req.count() as usize) { - debug!( - self.log, - "BlocksByRange outgoing response processed"; - "peer" => %peer_id, - "msg" => "Failed to return all requested blocks", - "start_slot" => req.start_slot(), - "current_slot" => current_slot, - "requested" => req.count(), - "returned" => blocks_sent - ); - } else { - debug!( - self.log, - "BlocksByRange outgoing response processed"; - "peer" => %peer_id, - "start_slot" => req.start_slot(), - "current_slot" => current_slot, - "requested" => req.count(), - "returned" => blocks_sent - ); - } - + log_results(req, peer_id, blocks_sent); Ok(()) } @@ -754,9 +758,25 @@ impl NetworkBeaconProcessor { } }; + let current_slot = self + .chain + .slot() + .unwrap_or_else(|_| self.chain.slot_clock.genesis_slot()); + + let log_results = |peer_id, req: BlobsByRangeRequest, blobs_sent| { + debug!( + self.log, + "BlobsByRange outgoing response processed"; + "peer" => %peer_id, + "start_slot" => req.start_slot, + "current_slot" => current_slot, + "requested" => req.count, + "returned" => blobs_sent + ); + }; + // remove all skip slots let block_roots = block_roots.into_iter().flatten(); - let mut blobs_sent = 0; for root in block_roots { @@ -780,6 +800,8 @@ impl NetworkBeaconProcessor { "block_root" => ?root, "error" => ?e ); + log_results(peer_id, req, blobs_sent); + return Err(( RPCResponseErrorCode::ServerError, "No blobs and failed fetching corresponding block", @@ -787,21 +809,7 @@ impl NetworkBeaconProcessor { } } } - - let current_slot = self - .chain - .slot() - .unwrap_or_else(|_| self.chain.slot_clock.genesis_slot()); - - debug!( - self.log, - "BlobsByRange Response processed"; - "peer" => %peer_id, - "start_slot" => req.start_slot, - "current_slot" => current_slot, - "requested" => req.count, - "returned" => blobs_sent - ); + log_results(peer_id, req, blobs_sent); Ok(()) } From cda926ce1b335055024bd705374f7a4e27f01dda Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Tue, 16 Apr 2024 14:03:08 -0500 Subject: [PATCH 09/37] Rename Functions to More Closely Match Spec (#5591) * Rename Functions to More Closely Match Spec --- beacon_node/genesis/src/eth1_genesis_service.rs | 4 ++-- .../state_processing/src/common/initiate_validator_exit.rs | 2 +- consensus/state_processing/src/genesis.rs | 4 ++-- .../src/per_block_processing/process_operations.rs | 4 ++-- .../src/per_epoch_processing/single_pass.rs | 2 +- consensus/types/src/beacon_state.rs | 6 +++--- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/beacon_node/genesis/src/eth1_genesis_service.rs b/beacon_node/genesis/src/eth1_genesis_service.rs index fdba9f4741c..0ede74ba754 100644 --- a/beacon_node/genesis/src/eth1_genesis_service.rs +++ b/beacon_node/genesis/src/eth1_genesis_service.rs @@ -5,7 +5,7 @@ use eth1::{DepositLog, Eth1Block, Service as Eth1Service}; use slog::{debug, error, info, trace, Logger}; use state_processing::{ eth2_genesis_time, initialize_beacon_state_from_eth1, is_valid_genesis_state, - per_block_processing::process_operations::process_deposit, process_activations, + per_block_processing::process_operations::apply_deposit, process_activations, }; use std::sync::{ atomic::{AtomicU64, AtomicUsize, Ordering}, @@ -433,7 +433,7 @@ impl Eth1GenesisService { // is reached _prior_ to `MIN_ACTIVE_VALIDATOR_COUNT`. I suspect this won't be the // case for mainnet, so we defer this optimization. - process_deposit(&mut state, &deposit, spec, PROOF_VERIFICATION) + apply_deposit(&mut state, &deposit, spec, PROOF_VERIFICATION) .map_err(|e| format!("Error whilst processing deposit: {:?}", e)) })?; diff --git a/consensus/state_processing/src/common/initiate_validator_exit.rs b/consensus/state_processing/src/common/initiate_validator_exit.rs index c527807df89..84656d9c890 100644 --- a/consensus/state_processing/src/common/initiate_validator_exit.rs +++ b/consensus/state_processing/src/common/initiate_validator_exit.rs @@ -26,7 +26,7 @@ pub fn initiate_validator_exit( .map_or(delayed_epoch, |epoch| max(epoch, delayed_epoch)); let exit_queue_churn = state.exit_cache().get_churn_at(exit_queue_epoch)?; - if exit_queue_churn >= state.get_churn_limit(spec)? { + if exit_queue_churn >= state.get_validator_churn_limit(spec)? { exit_queue_epoch.safe_add_assign(1)?; } diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index b225923b418..036ab23498c 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -1,5 +1,5 @@ use super::per_block_processing::{ - errors::BlockProcessingError, process_operations::process_deposit, + errors::BlockProcessingError, process_operations::apply_deposit, }; use crate::common::DepositDataTree; use crate::upgrade::{ @@ -37,7 +37,7 @@ pub fn initialize_beacon_state_from_eth1( .push_leaf(deposit.data.tree_hash_root()) .map_err(BlockProcessingError::MerkleTreeError)?; state.eth1_data_mut().deposit_root = deposit_tree.root(); - process_deposit(&mut state, deposit, spec, true)?; + apply_deposit(&mut state, deposit, spec, true)?; } process_activations(&mut state, spec)?; diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index af9b7938132..63b7c9e01fb 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -371,14 +371,14 @@ pub fn process_deposits( // Update the state in series. for deposit in deposits { - process_deposit(state, deposit, spec, false)?; + apply_deposit(state, deposit, spec, false)?; } Ok(()) } /// Process a single deposit, optionally verifying its merkle proof. -pub fn process_deposit( +pub fn apply_deposit( state: &mut BeaconState, deposit: &Deposit, spec: &ChainSpec, diff --git a/consensus/state_processing/src/per_epoch_processing/single_pass.rs b/consensus/state_processing/src/per_epoch_processing/single_pass.rs index 9319d2941b5..380484046c3 100644 --- a/consensus/state_processing/src/per_epoch_processing/single_pass.rs +++ b/consensus/state_processing/src/per_epoch_processing/single_pass.rs @@ -120,7 +120,7 @@ pub fn process_epoch_single_pass( let next_epoch = state.next_epoch()?; let is_in_inactivity_leak = state.is_in_inactivity_leak(previous_epoch, spec)?; let total_active_balance = state.get_total_active_balance()?; - let churn_limit = state.get_churn_limit(spec)?; + let churn_limit = state.get_validator_churn_limit(spec)?; let activation_churn_limit = state.get_activation_churn_limit(spec)?; let finalized_checkpoint = state.finalized_checkpoint(); let fork_name = state.fork_name_unchecked(); diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index ba11c9c4cce..02572b0efbd 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -1444,7 +1444,7 @@ impl BeaconState { /// Return the churn limit for the current epoch (number of validators who can leave per epoch). /// /// Uses the current epoch committee cache, and will error if it isn't initialized. - pub fn get_churn_limit(&self, spec: &ChainSpec) -> Result { + pub fn get_validator_churn_limit(&self, spec: &ChainSpec) -> Result { Ok(std::cmp::max( spec.min_per_epoch_churn_limit, (self @@ -1462,10 +1462,10 @@ impl BeaconState { BeaconState::Base(_) | BeaconState::Altair(_) | BeaconState::Merge(_) - | BeaconState::Capella(_) => self.get_churn_limit(spec)?, + | BeaconState::Capella(_) => self.get_validator_churn_limit(spec)?, BeaconState::Deneb(_) | BeaconState::Electra(_) => std::cmp::min( spec.max_per_epoch_activation_churn_limit, - self.get_churn_limit(spec)?, + self.get_validator_churn_limit(spec)?, ), }) } From 49617f3e8280a947e8ad2a3809e6b71da2165461 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 18 Apr 2024 01:09:09 +1000 Subject: [PATCH 10/37] Set web3signer keep-alive to 20s by default (#5587) * Set web3signer keep-alive to 20s by default * add tests --- book/src/help_vc.md | 2 +- lighthouse/tests/validator_client.rs | 26 +++++++++++++++++++++++++- validator_client/src/cli.rs | 2 +- validator_client/src/config.rs | 3 ++- validator_client/src/lib.rs | 2 +- 5 files changed, 30 insertions(+), 5 deletions(-) diff --git a/book/src/help_vc.md b/book/src/help_vc.md index fb963f87cc5..1b7e7f2b0af 100644 --- a/book/src/help_vc.md +++ b/book/src/help_vc.md @@ -218,7 +218,7 @@ OPTIONS: The directory which contains the validator keystores, deposit data for each validator along with the common slashing protection database and the validator_definitions.yml --web3-signer-keep-alive-timeout - Keep-alive timeout for each web3signer connection. Set to 'null' to never timeout [default: 90000] + Keep-alive timeout for each web3signer connection. Set to 'null' to never timeout [default: 20000] --web3-signer-max-idle-connections Maximum number of idle connections to maintain per web3signer host. Default is unlimited. diff --git a/lighthouse/tests/validator_client.rs b/lighthouse/tests/validator_client.rs index 764fd87ccdf..cdf8fa15aaa 100644 --- a/lighthouse/tests/validator_client.rs +++ b/lighthouse/tests/validator_client.rs @@ -1,4 +1,4 @@ -use validator_client::{ApiTopic, Config}; +use validator_client::{config::DEFAULT_WEB3SIGNER_KEEP_ALIVE, ApiTopic, Config}; use crate::exec::CommandLineTestExec; use bls::{Keypair, PublicKeyBytes}; @@ -9,6 +9,7 @@ use std::path::PathBuf; use std::process::Command; use std::str::FromStr; use std::string::ToString; +use std::time::Duration; use tempfile::TempDir; use types::Address; @@ -653,3 +654,26 @@ fn validator_disable_web3_signer_slashing_protection() { assert!(!config.enable_web3signer_slashing_protection); }); } + +#[test] +fn validator_web3_signer_keep_alive_default() { + CommandLineTest::new().run().with_config(|config| { + assert_eq!( + config.web3_signer_keep_alive_timeout, + DEFAULT_WEB3SIGNER_KEEP_ALIVE + ); + }); +} + +#[test] +fn validator_web3_signer_keep_alive_override() { + CommandLineTest::new() + .flag("web3-signer-keep-alive-timeout", Some("1000")) + .run() + .with_config(|config| { + assert_eq!( + config.web3_signer_keep_alive_timeout, + Some(Duration::from_secs(1)) + ); + }); +} diff --git a/validator_client/src/cli.rs b/validator_client/src/cli.rs index 16a265212e5..f91efbdfbc5 100644 --- a/validator_client/src/cli.rs +++ b/validator_client/src/cli.rs @@ -391,7 +391,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { Arg::with_name("web3-signer-keep-alive-timeout") .long("web3-signer-keep-alive-timeout") .value_name("MILLIS") - .default_value("90000") + .default_value("20000") .help("Keep-alive timeout for each web3signer connection. Set to 'null' to never \ timeout") .takes_value(true), diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index ae59829a3e6..5bd32fced2a 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -18,6 +18,7 @@ use std::time::Duration; use types::{Address, GRAFFITI_BYTES_LEN}; pub const DEFAULT_BEACON_NODE: &str = "http://localhost:5052/"; +pub const DEFAULT_WEB3SIGNER_KEEP_ALIVE: Option = Some(Duration::from_secs(20)); /// Stores the core configuration for this validator instance. #[derive(Clone, Serialize, Deserialize)] @@ -133,7 +134,7 @@ impl Default for Config { builder_boost_factor: None, prefer_builder_proposals: false, distributed: false, - web3_signer_keep_alive_timeout: Some(Duration::from_secs(90)), + web3_signer_keep_alive_timeout: DEFAULT_WEB3SIGNER_KEEP_ALIVE, web3_signer_max_idle_connections: None, } } diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 32a0eadbef4..268c25cdf7d 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -3,7 +3,6 @@ mod beacon_node_fallback; mod block_service; mod check_synced; mod cli; -mod config; mod duties_service; mod graffiti_file; mod http_metrics; @@ -14,6 +13,7 @@ mod preparation_service; mod signing_method; mod sync_committee_service; +pub mod config; mod doppelganger_service; pub mod http_api; pub mod initialized_validators; From 62e4abfbff1b06192a34751cf849d1547b2d5ef6 Mon Sep 17 00:00:00 2001 From: AMIT SINGH Date: Thu, 18 Apr 2024 12:44:56 +0530 Subject: [PATCH 11/37] Fix execution layer redundancy (#5588) * remove execution layer url redundancy * fix typo * fix tests * fix formatting --- beacon_node/beacon_chain/src/test_utils.rs | 12 ++++------ beacon_node/execution_layer/src/lib.rs | 22 +++++++------------ .../src/test_utils/mock_builder.rs | 4 ++-- .../src/test_utils/mock_execution_layer.rs | 4 ++-- beacon_node/src/config.rs | 4 ++-- lighthouse/tests/beacon_node.rs | 20 +++++++++++------ .../src/test_rig.rs | 12 +++++----- testing/simulator/src/eth1_sim.rs | 8 +++---- testing/simulator/src/local_network.rs | 12 +++++----- 9 files changed, 46 insertions(+), 52 deletions(-) diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 542262487ae..debc4881a60 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -419,21 +419,17 @@ where self } - pub fn execution_layer_from_urls(mut self, urls: &[&str]) -> Self { + pub fn execution_layer_from_url(mut self, url: &str) -> Self { assert!( self.execution_layer.is_none(), "execution layer already defined" ); - let urls: Vec = urls - .iter() - .map(|s| SensitiveUrl::parse(s)) - .collect::>() - .unwrap(); + let url = SensitiveUrl::parse(url).ok(); let config = execution_layer::Config { - execution_endpoints: urls, - secret_files: vec![], + execution_endpoint: url, + secret_file: None, suggested_fee_recipient: Some(Address::repeat_byte(42)), ..Default::default() }; diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 30930318eff..22410976c9d 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -355,14 +355,14 @@ struct Inner { #[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct Config { - /// Endpoint urls for EL nodes that are running the engine api. - pub execution_endpoints: Vec, + /// Endpoint url for EL nodes that are running the engine api. + pub execution_endpoint: Option, /// Endpoint urls for services providing the builder api. pub builder_url: Option, /// User agent to send with requests to the builder API. pub builder_user_agent: Option, - /// JWT secrets for the above endpoints running the engine api. - pub secret_files: Vec, + /// JWT secret for the above endpoint running the engine api. + pub secret_file: Option, /// The default fee recipient to use on the beacon node if none if provided from /// the validator client during block preparation. pub suggested_fee_recipient: Option
, @@ -386,10 +386,10 @@ impl ExecutionLayer { /// Instantiate `Self` with an Execution engine specified in `Config`, using JSON-RPC via HTTP. pub fn from_config(config: Config, executor: TaskExecutor, log: Logger) -> Result { let Config { - execution_endpoints: urls, + execution_endpoint: url, builder_url, builder_user_agent, - secret_files, + secret_file, suggested_fee_recipient, jwt_id, jwt_version, @@ -397,16 +397,10 @@ impl ExecutionLayer { execution_timeout_multiplier, } = config; - if urls.len() > 1 { - warn!(log, "Only the first execution engine url will be used"); - } - let execution_url = urls.into_iter().next().ok_or(Error::NoEngine)?; + let execution_url = url.ok_or(Error::NoEngine)?; // Use the default jwt secret path if not provided via cli. - let secret_file = secret_files - .into_iter() - .next() - .unwrap_or_else(|| default_datadir.join(DEFAULT_JWT_FILE)); + let secret_file = secret_file.unwrap_or_else(|| default_datadir.join(DEFAULT_JWT_FILE)); let jwt_key = if secret_file.exists() { // Read secret from file if it already exists diff --git a/beacon_node/execution_layer/src/test_utils/mock_builder.rs b/beacon_node/execution_layer/src/test_utils/mock_builder.rs index b12e26a3d6c..756e0b793f8 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_builder.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_builder.rs @@ -229,8 +229,8 @@ impl MockBuilder { // This EL should not talk to a builder let config = Config { - execution_endpoints: vec![mock_el_url], - secret_files: vec![path], + execution_endpoint: Some(mock_el_url), + secret_file: Some(path), suggested_fee_recipient: None, ..Default::default() }; diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index f76edfa90b7..6717bbc2ab3 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -67,8 +67,8 @@ impl MockExecutionLayer { std::fs::write(&path, hex::encode(DEFAULT_JWT_SECRET)).unwrap(); let config = Config { - execution_endpoints: vec![url], - secret_files: vec![path], + execution_endpoint: Some(url), + secret_file: Some(path), suggested_fee_recipient: Some(Address::repeat_byte(42)), ..Default::default() }; diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 5a27b148c99..fd2cf473cb3 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -348,8 +348,8 @@ pub fn get_config( } // Set config values from parse values. - el_config.secret_files = vec![secret_file.clone()]; - el_config.execution_endpoints = vec![execution_endpoint.clone()]; + el_config.secret_file = Some(secret_file.clone()); + el_config.execution_endpoint = Some(execution_endpoint.clone()); el_config.suggested_fee_recipient = clap_utils::parse_optional(cli_args, "suggested-fee-recipient")?; el_config.jwt_id = clap_utils::parse_optional(cli_args, "execution-jwt-id")?; diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 4a50126945b..ec10ff4429d 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -445,13 +445,16 @@ fn run_merge_execution_endpoints_flag_test(flag: &str) { .run_with_zero_port() .with_config(|config| { let config = config.execution_layer.as_ref().unwrap(); - assert_eq!(config.execution_endpoints.len(), 1); + assert_eq!(config.execution_endpoint.is_some(), true); assert_eq!( - config.execution_endpoints[0], + config.execution_endpoint.as_ref().unwrap().clone(), SensitiveUrl::parse(&urls[0]).unwrap() ); // Only the first secret file should be used. - assert_eq!(config.secret_files, vec![jwts[0].clone()]); + assert_eq!( + config.secret_file.as_ref().unwrap().clone(), + jwts[0].clone() + ); }); } #[test] @@ -464,11 +467,11 @@ fn run_execution_jwt_secret_key_is_persisted() { .with_config(|config| { let config = config.execution_layer.as_ref().unwrap(); assert_eq!( - config.execution_endpoints[0].full.to_string(), + config.execution_endpoint.as_ref().unwrap().full.to_string(), "http://localhost:8551/" ); let mut file_jwt_secret_key = String::new(); - File::open(config.secret_files[0].clone()) + File::open(config.secret_file.as_ref().unwrap()) .expect("could not open jwt_secret_key file") .read_to_string(&mut file_jwt_secret_key) .expect("could not read from file"); @@ -515,10 +518,13 @@ fn merge_jwt_secrets_flag() { .with_config(|config| { let config = config.execution_layer.as_ref().unwrap(); assert_eq!( - config.execution_endpoints[0].full.to_string(), + config.execution_endpoint.as_ref().unwrap().full.to_string(), "http://localhost:8551/" ); - assert_eq!(config.secret_files[0], dir.path().join("jwt-file")); + assert_eq!( + config.secret_file.as_ref().unwrap().clone(), + dir.path().join("jwt-file") + ); }); } #[test] diff --git a/testing/execution_engine_integration/src/test_rig.rs b/testing/execution_engine_integration/src/test_rig.rs index 8f782c7e4e0..0103f7074b5 100644 --- a/testing/execution_engine_integration/src/test_rig.rs +++ b/testing/execution_engine_integration/src/test_rig.rs @@ -121,11 +121,11 @@ impl TestRig { let ee_a = { let execution_engine = ExecutionEngine::new(generic_engine.clone()); - let urls = vec![execution_engine.http_auth_url()]; + let url = Some(execution_engine.http_auth_url()); let config = execution_layer::Config { - execution_endpoints: urls, - secret_files: vec![], + execution_endpoint: url, + secret_file: None, suggested_fee_recipient: Some(Address::repeat_byte(42)), default_datadir: execution_engine.datadir(), ..Default::default() @@ -140,11 +140,11 @@ impl TestRig { let ee_b = { let execution_engine = ExecutionEngine::new(generic_engine); - let urls = vec![execution_engine.http_auth_url()]; + let url = Some(execution_engine.http_auth_url()); let config = execution_layer::Config { - execution_endpoints: urls, - secret_files: vec![], + execution_endpoint: url, + secret_file: None, suggested_fee_recipient: fee_recipient, default_datadir: execution_engine.datadir(), ..Default::default() diff --git a/testing/simulator/src/eth1_sim.rs b/testing/simulator/src/eth1_sim.rs index 8d6ffc42ffa..20c7c9ce9ab 100644 --- a/testing/simulator/src/eth1_sim.rs +++ b/testing/simulator/src/eth1_sim.rs @@ -395,11 +395,9 @@ async fn create_local_network( if post_merge_sim { let el_config = execution_layer::Config { - execution_endpoints: vec![SensitiveUrl::parse(&format!( - "http://localhost:{}", - EXECUTION_PORT - )) - .unwrap()], + execution_endpoint: Some( + SensitiveUrl::parse(&format!("http://localhost:{}", EXECUTION_PORT)).unwrap(), + ), ..Default::default() }; diff --git a/testing/simulator/src/local_network.rs b/testing/simulator/src/local_network.rs index dc8bf0d27dd..018954a5d3b 100644 --- a/testing/simulator/src/local_network.rs +++ b/testing/simulator/src/local_network.rs @@ -85,9 +85,9 @@ impl LocalNetwork { mock_execution_config, ); el_config.default_datadir = execution_node.datadir.path().to_path_buf(); - el_config.secret_files = vec![execution_node.datadir.path().join("jwt.hex")]; - el_config.execution_endpoints = - vec![SensitiveUrl::parse(&execution_node.server.url()).unwrap()]; + el_config.secret_file = Some(execution_node.datadir.path().join("jwt.hex")); + el_config.execution_endpoint = + Some(SensitiveUrl::parse(&execution_node.server.url()).unwrap()); vec![execution_node] } else { vec![] @@ -180,9 +180,9 @@ impl LocalNetwork { config, ); el_config.default_datadir = execution_node.datadir.path().to_path_buf(); - el_config.secret_files = vec![execution_node.datadir.path().join("jwt.hex")]; - el_config.execution_endpoints = - vec![SensitiveUrl::parse(&execution_node.server.url()).unwrap()]; + el_config.secret_file = Some(execution_node.datadir.path().join("jwt.hex")); + el_config.execution_endpoint = + Some(SensitiveUrl::parse(&execution_node.server.url()).unwrap()); self.execution_nodes.write().push(execution_node); } From 5c30afbc7c0ef32fc70236543298a0db0ad5bd71 Mon Sep 17 00:00:00 2001 From: chonghe <44791194+chong-he@users.noreply.github.com> Date: Thu, 18 Apr 2024 15:14:59 +0800 Subject: [PATCH 12/37] Revise `secrets-dir` flag in the VC (#5480) * Update docs on secrets-dir * Hidden secrets-dir flag * Remove conflicts_with * Restore description * make cli * Update book/src/validator-management.md --- account_manager/src/validator/create.rs | 1 - book/src/validator-management.md | 5 ++++- validator_client/src/cli.rs | 1 - 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/account_manager/src/validator/create.rs b/account_manager/src/validator/create.rs index 8da32531a80..93b041c61c4 100644 --- a/account_manager/src/validator/create.rs +++ b/account_manager/src/validator/create.rs @@ -62,7 +62,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { "The path where the validator keystore passwords will be stored. \ Defaults to ~/.lighthouse/{network}/secrets", ) - .conflicts_with("datadir") .takes_value(true), ) .arg( diff --git a/book/src/validator-management.md b/book/src/validator-management.md index df7c2ac4760..bc6aba3c4f9 100644 --- a/book/src/validator-management.md +++ b/book/src/validator-management.md @@ -59,7 +59,9 @@ Each permitted field of the file is listed below for reference: - `voting_keystore_password`: The password to the EIP-2335 keystore. > **Note**: Either `voting_keystore_password_path` or `voting_keystore_password` *must* be -> supplied. If both are supplied, `voting_keystore_password_path` is ignored. +> supplied. If both are supplied, `voting_keystore_password_path` is ignored. + +>If you do not wish to have `voting_keystore_password` being stored in the `validator_definitions.yml` file, you can add the field `voting_keystore_password_path` and point it to a file containing the password. The file can be, e.g., on a mounted portable drive that contains the password so that no password is stored on the validating node. ## Populating the `validator_definitions.yml` file @@ -75,6 +77,7 @@ recap: ### Automatic validator discovery + When the `--disable-auto-discover` flag is **not** provided, the validator client will search the `validator-dir` for validators and add any *new* validators to the `validator_definitions.yml` with `enabled: true`. diff --git a/validator_client/src/cli.rs b/validator_client/src/cli.rs index f91efbdfbc5..991b621f27d 100644 --- a/validator_client/src/cli.rs +++ b/validator_client/src/cli.rs @@ -73,7 +73,6 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { key. Defaults to ~/.lighthouse/{network}/secrets.", ) .takes_value(true) - .conflicts_with("datadir") ) .arg( Arg::with_name("init-slashing-protection") From 5a9e973f047fd7c401339ff8bd715f43bab8a1a0 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Fri, 19 Apr 2024 18:18:30 +1000 Subject: [PATCH 13/37] Fix on-disk consensus context format (#5598) * Fix on-disk consensus context format * Keep indexed attestations, thanks Sean --- .../state_lru_cache.rs | 21 ++++-- beacon_node/store/src/consensus_context.rs | 66 +++++++++++++++++++ beacon_node/store/src/lib.rs | 2 + .../state_processing/src/consensus_context.rs | 31 ++++++--- 4 files changed, 104 insertions(+), 16 deletions(-) create mode 100644 beacon_node/store/src/consensus_context.rs diff --git a/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs index c3492b53bda..b6dbf2b952f 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs @@ -8,8 +8,9 @@ use crate::{ use lru::LruCache; use parking_lot::RwLock; use ssz_derive::{Decode, Encode}; -use state_processing::{BlockReplayer, ConsensusContext, StateProcessingStrategy}; +use state_processing::{BlockReplayer, StateProcessingStrategy}; use std::sync::Arc; +use store::OnDiskConsensusContext; use types::beacon_block_body::KzgCommitments; use types::{ssz_tagged_signed_beacon_block, ssz_tagged_signed_beacon_block_arc}; use types::{BeaconState, BlindedPayload, ChainSpec, Epoch, EthSpec, Hash256, SignedBeaconBlock}; @@ -26,7 +27,7 @@ pub struct DietAvailabilityPendingExecutedBlock { parent_block: SignedBeaconBlock>, parent_eth1_finalization_data: Eth1FinalizationData, confirmed_state_roots: Vec, - consensus_context: ConsensusContext, + consensus_context: OnDiskConsensusContext, payload_verification_outcome: PayloadVerificationOutcome, } @@ -94,7 +95,9 @@ impl StateLRUCache { parent_block: executed_block.import_data.parent_block, parent_eth1_finalization_data: executed_block.import_data.parent_eth1_finalization_data, confirmed_state_roots: executed_block.import_data.confirmed_state_roots, - consensus_context: executed_block.import_data.consensus_context, + consensus_context: OnDiskConsensusContext::from_consensus_context( + executed_block.import_data.consensus_context, + ), payload_verification_outcome: executed_block.payload_verification_outcome, } } @@ -119,7 +122,9 @@ impl StateLRUCache { parent_eth1_finalization_data: diet_executed_block .parent_eth1_finalization_data, confirmed_state_roots: diet_executed_block.confirmed_state_roots, - consensus_context: diet_executed_block.consensus_context, + consensus_context: diet_executed_block + .consensus_context + .into_consensus_context(), }, payload_verification_outcome: diet_executed_block.payload_verification_outcome, }) @@ -145,7 +150,9 @@ impl StateLRUCache { parent_block: diet_executed_block.parent_block, parent_eth1_finalization_data: diet_executed_block.parent_eth1_finalization_data, confirmed_state_roots: diet_executed_block.confirmed_state_roots, - consensus_context: diet_executed_block.consensus_context, + consensus_context: diet_executed_block + .consensus_context + .into_consensus_context(), }, payload_verification_outcome: diet_executed_block.payload_verification_outcome, }) @@ -232,7 +239,9 @@ impl From> parent_block: value.import_data.parent_block, parent_eth1_finalization_data: value.import_data.parent_eth1_finalization_data, confirmed_state_roots: value.import_data.confirmed_state_roots, - consensus_context: value.import_data.consensus_context, + consensus_context: OnDiskConsensusContext::from_consensus_context( + value.import_data.consensus_context, + ), payload_verification_outcome: value.payload_verification_outcome, } } diff --git a/beacon_node/store/src/consensus_context.rs b/beacon_node/store/src/consensus_context.rs new file mode 100644 index 00000000000..08fad17b14b --- /dev/null +++ b/beacon_node/store/src/consensus_context.rs @@ -0,0 +1,66 @@ +use ssz_derive::{Decode, Encode}; +use state_processing::ConsensusContext; +use std::collections::HashMap; +use types::{AttestationData, BitList, EthSpec, Hash256, IndexedAttestation, Slot}; + +/// The consensus context is stored on disk as part of the data availability overflow cache. +/// +/// We use this separate struct to keep the on-disk format stable in the presence of changes to the +/// in-memory `ConsensusContext`. You MUST NOT change the fields of this struct without +/// superstructing it and implementing a schema migration. +#[derive(Debug, PartialEq, Clone, Encode, Decode)] +pub struct OnDiskConsensusContext { + /// Slot to act as an identifier/safeguard + slot: Slot, + /// Proposer index of the block at `slot`. + proposer_index: Option, + /// Block root of the block at `slot`. + current_block_root: Option, + /// We keep the indexed attestations in the *in-memory* version of this struct so that we don't + /// need to regenerate them if roundtripping via this type *without* going to disk. + /// + /// They are not part of the on-disk format. + #[ssz(skip_serializing, skip_deserializing)] + indexed_attestations: + HashMap<(AttestationData, BitList), IndexedAttestation>, +} + +impl OnDiskConsensusContext { + pub fn from_consensus_context(ctxt: ConsensusContext) -> Self { + // Match exhaustively on fields here so we are forced to *consider* updating the on-disk + // format when the `ConsensusContext` fields change. + let ConsensusContext { + slot, + previous_epoch: _, + current_epoch: _, + proposer_index, + current_block_root, + indexed_attestations, + } = ctxt; + OnDiskConsensusContext { + slot, + proposer_index, + current_block_root, + indexed_attestations, + } + } + + pub fn into_consensus_context(self) -> ConsensusContext { + let OnDiskConsensusContext { + slot, + proposer_index, + current_block_root, + indexed_attestations, + } = self; + + let mut ctxt = ConsensusContext::new(slot); + + if let Some(proposer_index) = proposer_index { + ctxt = ctxt.set_proposer_index(proposer_index); + } + if let Some(block_root) = current_block_root { + ctxt = ctxt.set_current_block_root(block_root); + } + ctxt.set_indexed_attestations(indexed_attestations) + } +} diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index e86689b0cf1..c3136a910db 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -14,6 +14,7 @@ mod chunk_writer; pub mod chunked_iter; pub mod chunked_vector; pub mod config; +pub mod consensus_context; pub mod errors; mod forwards_iter; mod garbage_collection; @@ -30,6 +31,7 @@ pub mod iter; pub use self::chunk_writer::ChunkWriter; pub use self::config::StoreConfig; +pub use self::consensus_context::OnDiskConsensusContext; pub use self::hot_cold_store::{HotColdDB, HotStateSummary, Split}; pub use self::leveldb_store::LevelDB; pub use self::memory_store::MemoryStore; diff --git a/consensus/state_processing/src/consensus_context.rs b/consensus/state_processing/src/consensus_context.rs index 263539fa429..073d87be85b 100644 --- a/consensus/state_processing/src/consensus_context.rs +++ b/consensus/state_processing/src/consensus_context.rs @@ -1,7 +1,6 @@ use crate::common::get_indexed_attestation; use crate::per_block_processing::errors::{AttestationInvalid, BlockOperationError}; use crate::EpochCacheError; -use ssz_derive::{Decode, Encode}; use std::collections::{hash_map::Entry, HashMap}; use tree_hash::TreeHash; use types::{ @@ -9,22 +8,20 @@ use types::{ ChainSpec, Epoch, EthSpec, Hash256, IndexedAttestation, SignedBeaconBlock, Slot, }; -#[derive(Debug, PartialEq, Clone, Encode, Decode)] +#[derive(Debug, PartialEq, Clone)] pub struct ConsensusContext { /// Slot to act as an identifier/safeguard - slot: Slot, + pub slot: Slot, /// Previous epoch of the `slot` precomputed for optimization purpose. - pub(crate) previous_epoch: Epoch, + pub previous_epoch: Epoch, /// Current epoch of the `slot` precomputed for optimization purpose. - pub(crate) current_epoch: Epoch, + pub current_epoch: Epoch, /// Proposer index of the block at `slot`. - proposer_index: Option, + pub proposer_index: Option, /// Block root of the block at `slot`. - current_block_root: Option, + pub current_block_root: Option, /// Cache of indexed attestations constructed during block processing. - /// We can skip serializing / deserializing this as the cache will just be rebuilt - #[ssz(skip_serializing, skip_deserializing)] - indexed_attestations: + pub indexed_attestations: HashMap<(AttestationData, BitList), IndexedAttestation>, } @@ -62,6 +59,7 @@ impl ConsensusContext { } } + #[must_use] pub fn set_proposer_index(mut self, proposer_index: u64) -> Self { self.proposer_index = Some(proposer_index); self @@ -109,6 +107,7 @@ impl ConsensusContext { Ok(proposer_index) } + #[must_use] pub fn set_current_block_root(mut self, block_root: Hash256) -> Self { self.current_block_root = Some(block_root); self @@ -174,4 +173,16 @@ impl ConsensusContext { pub fn num_cached_indexed_attestations(&self) -> usize { self.indexed_attestations.len() } + + #[must_use] + pub fn set_indexed_attestations( + mut self, + attestations: HashMap< + (AttestationData, BitList), + IndexedAttestation, + >, + ) -> Self { + self.indexed_attestations = attestations; + self + } } From 9b5895ca8969a6b76e52e9d0e15445b2c6f69404 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Mon, 22 Apr 2024 14:01:06 +0100 Subject: [PATCH 14/37] Fix cargo audit RUSTSEC-2024-0336 (#5612) * replace unmaintained rust_yaml with serde_yml * update warp --- Cargo.lock | 1199 +++++++++++++++++++---------- Cargo.toml | 6 +- consensus/int_to_bytes/Cargo.toml | 2 +- consensus/int_to_bytes/src/lib.rs | 10 +- 4 files changed, 821 insertions(+), 396 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0452fa0e8c7..79be0a55f18 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,7 +18,7 @@ version = "0.3.5" dependencies = [ "account_utils", "bls", - "clap", + "clap 2.34.0", "clap_utils", "directory", "environment", @@ -126,14 +126,14 @@ dependencies = [ [[package]] name = "aes-gcm" -version = "0.9.4" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df5f85a83a7d8b0442b6aa7b504b8212c1733da07b98aae43d4bc21b2cb3cdf6" +checksum = "bc3be92e19a7ef47457b8e6f90707e12b6ac5d20c6f3866584fa3be0787d839f" dependencies = [ "aead 0.4.3", "aes 0.7.5", "cipher 0.3.0", - "ctr 0.8.0", + "ctr 0.7.0", "ghash 0.4.4", "subtle", ] @@ -148,15 +148,15 @@ dependencies = [ "aes 0.8.4", "cipher 0.4.4", "ctr 0.9.2", - "ghash 0.5.0", + "ghash 0.5.1", "subtle", ] [[package]] name = "ahash" -version = "0.8.10" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b79b82693f705137f8fb9b37871d99e4f9a7df12b917eed79c3d3954830a60b" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" dependencies = [ "cfg-if", "once_cell", @@ -166,18 +166,18 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.2" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" dependencies = [ "memchr", ] [[package]] name = "allocator-api2" -version = "0.2.16" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" [[package]] name = "alloy-consensus" @@ -265,7 +265,7 @@ checksum = "1a047897373be4bbb0224c1afdabca92648dc57a9c9ef6e7b0be3aff7a859c83" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -292,11 +292,59 @@ dependencies = [ "winapi", ] +[[package]] +name = "anstream" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" + +[[package]] +name = "anstyle-parse" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + [[package]] name = "anyhow" -version = "1.0.80" +version = "1.0.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" +checksum = "f538837af36e6f6a9be0faa67f9a314f8119e4e4b5867c6ab40ed60360142519" [[package]] name = "arbitrary" @@ -309,9 +357,9 @@ dependencies = [ [[package]] name = "arc-swap" -version = "1.6.0" +version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" [[package]] name = "ark-ff" @@ -507,9 +555,9 @@ dependencies = [ [[package]] name = "async-io" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f97ab0c5b00a7cdbe5a371b9a782ee7be1316095885c8a4ea1daf490eb0ef65" +checksum = "dcccb0f599cfa2f8ace422d3555572f47424da5648a4382a9dd0310ff8210884" dependencies = [ "async-lock", "cfg-if", @@ -518,7 +566,7 @@ dependencies = [ "futures-lite", "parking", "polling", - "rustix 0.38.31", + "rustix 0.38.33", "slab", "tracing", "windows-sys 0.52.0", @@ -537,13 +585,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.77" +version = "0.1.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -589,7 +637,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d9a9bf8b79a749ee0b911b91b671cc2b6c670bdbc7e3dfd537576ddc94bb2a2" dependencies = [ - "http 0.2.11", + "http 0.2.12", "log", "url", ] @@ -613,29 +661,29 @@ checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] name = "autocfg" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +checksum = "f1fdabc7756949593fe60f30ec81974b613357de856987752631dea1e3394c80" [[package]] name = "axum" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1236b4b292f6c4d6dc34604bb5120d85c3fe1d1aa596bd5cc52ca054d13e7b9e" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" dependencies = [ "async-trait", "axum-core", "bytes", "futures-util", - "http 1.0.0", + "http 1.1.0", "http-body 1.0.0", "http-body-util", - "hyper 1.2.0", + "hyper 1.3.1", "hyper-util", "itoa", "matchit", @@ -648,7 +696,7 @@ dependencies = [ "serde_json", "serde_path_to_error", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 1.0.1", "tokio", "tower", "tower-layer", @@ -665,13 +713,13 @@ dependencies = [ "async-trait", "bytes", "futures-util", - "http 1.0.0", + "http 1.1.0", "http-body 1.0.0", "http-body-util", "mime", "pin-project-lite", "rustversion", - "sync_wrapper", + "sync_wrapper 0.1.2", "tower-layer", "tower-service", "tracing", @@ -679,9 +727,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.69" +version = "0.3.71" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +checksum = "26b05800d2e817c8b3b4b54abd461726265fa9789ae34330622f2db9ee696f9d" dependencies = [ "addr2line", "cc", @@ -722,6 +770,12 @@ version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" +[[package]] +name = "base64" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" + [[package]] name = "base64ct" version = "1.6.0" @@ -793,7 +847,7 @@ name = "beacon_node" version = "5.1.3" dependencies = [ "beacon_chain", - "clap", + "clap 2.34.0", "clap_utils", "client", "directory", @@ -806,7 +860,7 @@ dependencies = [ "genesis", "hex", "http_api", - "hyper 1.2.0", + "hyper 1.3.1", "lighthouse_network", "lighthouse_version", "monitoring_api", @@ -900,9 +954,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.2" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" [[package]] name = "bitvec" @@ -1006,7 +1060,7 @@ name = "boot_node" version = "5.1.3" dependencies = [ "beacon_node", - "clap", + "clap 2.34.0", "clap_utils", "eth2_network_config", "ethereum_ssz", @@ -1034,9 +1088,9 @@ checksum = "771fe0050b883fcc3ea2359b1a96bcfbc090b7116eae7c3c512c7a083fdf23d3" [[package]] name = "bs58" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5353f36341f7451062466f0b755b96ac3a9547e4d7f6b70d603fc721a7d7896" +checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" dependencies = [ "tinyvec", ] @@ -1055,9 +1109,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.15.3" +version = "3.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ea184aa71bb362a1157c896979544cc23974e08fd265f29ea96b59f0b4a555b" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" [[package]] name = "byte-slice-cast" @@ -1073,9 +1127,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" [[package]] name = "bytes" -version = "1.5.0" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2bd12c1caf447e69cd4528f47f94d203fd2582878ecb9e9465484c4148a8223" +checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" dependencies = [ "serde", ] @@ -1141,9 +1195,9 @@ dependencies = [ [[package]] name = "cargo-platform" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "694c8807f2ae16faecc43dc17d74b3eb042482789fd0eb64b39a2e04e087053f" +checksum = "24b1f0365a6c6bb4020cd05806fd0d33c44d38046b8bd7f0e40814b9763cabfc" dependencies = [ "serde", ] @@ -1170,12 +1224,13 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.0.90" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cd6604a82acf3039f1144f54b8eb34e91ffba622051189e71b781822d5ee1f5" +checksum = "d32a725bc159af97c3e629873bb9f88fb8cf8a4867175f76dc987815ea07c83b" dependencies = [ "jobserver", "libc", + "once_cell", ] [[package]] @@ -1193,6 +1248,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfg_aliases" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd16c4719339c4530435d38e511904438d07cce7950afa3718a84ac36c10e89e" + [[package]] name = "chacha20" version = "0.9.1" @@ -1219,14 +1280,14 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.34" +version = "0.4.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" dependencies = [ "android-tzdata", "iana-time-zone", "num-traits", - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -1275,11 +1336,38 @@ dependencies = [ "vec_map", ] +[[package]] +name = "clap" +version = "4.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim 0.11.1", +] + +[[package]] +name = "clap_lex" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" + [[package]] name = "clap_utils" version = "0.1.0" dependencies = [ - "clap", + "clap 2.34.0", "dirs", "eth2_network_config", "ethereum-types 0.14.1", @@ -1344,6 +1432,12 @@ dependencies = [ "cc", ] +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "compare_fields" version = "0.2.0" @@ -1369,11 +1463,24 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.15.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" +dependencies = [ + "encode_unicode", + "lazy_static", + "libc", + "unicode-width", + "windows-sys 0.52.0", +] + [[package]] name = "const-hex" -version = "1.11.1" +version = "1.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efbd12d49ab0eaf8193ba9175e45f56bbc2e4b27d57b8cfe62aa47942a46b9a9" +checksum = "5ba00838774b4ab0233e355d26710fbfc8327a05c017f6dc4873f876d1f79f78" dependencies = [ "cfg-if", "cpufeatures", @@ -1451,7 +1558,7 @@ checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" dependencies = [ "atty", "cast", - "clap", + "clap 2.34.0", "criterion-plot", "csv", "itertools", @@ -1566,9 +1673,9 @@ dependencies = [ [[package]] name = "crypto-mac" -version = "0.11.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" +checksum = "25fab6889090c8133f3deb8f73ba3c65a7f456f66436fc012a1b1e272b1e103e" dependencies = [ "generic-array", "subtle", @@ -1595,6 +1702,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "ctr" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a232f92a03f37dd7d7dd2adc67166c77e9cd88de5b019b9a9eecfaeaf7bfd481" +dependencies = [ + "cipher 0.3.0", +] + [[package]] name = "ctr" version = "0.8.0" @@ -1615,11 +1731,11 @@ dependencies = [ [[package]] name = "ctrlc" -version = "3.4.2" +version = "3.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b467862cc8610ca6fc9a1532d7777cee0804e678ab45410897b9396495994a0b" +checksum = "672465ae37dc1bc6380a6547a8883d5dd397b0f1faaad4f265726cc7042a5345" dependencies = [ - "nix 0.27.1", + "nix 0.28.0", "windows-sys 0.52.0", ] @@ -1634,7 +1750,7 @@ dependencies = [ "curve25519-dalek-derive", "digest 0.10.7", "fiat-crypto", - "platforms 3.3.0", + "platforms 3.4.0", "rustc_version 0.4.0", "subtle", "zeroize", @@ -1648,7 +1764,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -1657,8 +1773,18 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.13.4", + "darling_macro 0.13.4", +] + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", ] [[package]] @@ -1675,13 +1801,38 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", +] + [[package]] name = "darling_macro" version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ - "darling_core", + "darling_core 0.13.4", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", "quote", "syn 1.0.109", ] @@ -1744,7 +1895,7 @@ version = "0.1.0" dependencies = [ "beacon_chain", "beacon_node", - "clap", + "clap 2.34.0", "clap_utils", "environment", "hex", @@ -1799,9 +1950,9 @@ dependencies = [ [[package]] name = "der" -version = "0.7.8" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fffa369a668c8af7dbf8b5e56c9f744fbd399949ed171606040001947de40b1c" +checksum = "f55bf8e7b65898637379c1b74eb1551107c8294ed26d855ceb9fd1a09cfc9bc0" dependencies = [ "const-oid", "pem-rfc7468", @@ -1850,7 +2001,38 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", +] + +[[package]] +name = "derive_builder" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "derive_builder_macro" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" +dependencies = [ + "derive_builder_core", + "syn 1.0.109", ] [[package]] @@ -1866,13 +2048,26 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "dialoguer" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "658bce805d770f407bc62102fca7c2c64ceef2fbcb2b8bd19d2765ce093980de" +dependencies = [ + "console", + "shell-words", + "tempfile", + "thiserror", + "zeroize", +] + [[package]] name = "diesel" -version = "2.1.4" +version = "2.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62c6fcf842f17f8c78ecf7c81d75c5ce84436b41ee07e03f490fbb5f5a8731d8" +checksum = "ff236accb9a5069572099f0b350a92e9560e8e63a9b8d546162f4a5e03026bb2" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "byteorder", "diesel_derives", "itoa", @@ -1882,14 +2077,14 @@ dependencies = [ [[package]] name = "diesel_derives" -version = "2.1.2" +version = "2.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8337737574f55a468005a83499da720f20c65586241ffea339db9ecdfd2b44" +checksum = "14701062d6bed917b5c7103bdffaee1e4609279e240488ad24e7bd979ca6866c" dependencies = [ "diesel_table_macro_syntax", "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -1909,7 +2104,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" dependencies = [ - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -1937,7 +2132,7 @@ dependencies = [ name = "directory" version = "0.1.0" dependencies = [ - "clap", + "clap 2.34.0", "clap_utils", "eth2_network_config", ] @@ -1990,7 +2185,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bac33cb3f99889a57e56a8c6ccb77aaf0cfc7787602b7af09783f736d77314e1" dependencies = [ "aes 0.7.5", - "aes-gcm 0.9.4", + "aes-gcm 0.9.2", "arrayvec", "delay_map", "enr", @@ -2022,7 +2217,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -2031,6 +2226,29 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" +[[package]] +name = "dtt" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6b2dd9ee2d76888dc4c17d6da74629fa11b3cb1e8094fdc159b7f8ff259fc88" +dependencies = [ + "regex", + "serde", + "time", +] + +[[package]] +name = "duct" +version = "0.13.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c" +dependencies = [ + "libc", + "once_cell", + "os_pipe", + "shared_child", +] + [[package]] name = "dunce" version = "1.0.4" @@ -2055,7 +2273,7 @@ version = "0.16.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" dependencies = [ - "der 0.7.8", + "der 0.7.9", "digest 0.10.7", "elliptic-curve 0.13.8", "rfc6979 0.4.0", @@ -2125,9 +2343,9 @@ dependencies = [ [[package]] name = "either" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" +checksum = "a47c1c47d2f5964e29c61246e81db715514cd532db6b5116a25ea3c03d6780a2" [[package]] name = "elliptic-curve" @@ -2168,11 +2386,17 @@ dependencies = [ "zeroize", ] +[[package]] +name = "encode_unicode" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" + [[package]] name = "encoding_rs" -version = "0.8.33" +version = "0.8.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7268b386296a025e474d5140678f75d6de9493ae55a5d709eeb9dd08149945e1" +checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59" dependencies = [ "cfg-if", ] @@ -2205,7 +2429,17 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", +] + +[[package]] +name = "env_filter" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" +dependencies = [ + "log", + "regex", ] [[package]] @@ -2231,6 +2465,19 @@ dependencies = [ "termcolor", ] +[[package]] +name = "env_logger" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "humantime", + "log", +] + [[package]] name = "environment" version = "0.1.2" @@ -2418,7 +2665,7 @@ dependencies = [ "sha2 0.9.9", "tempfile", "unicode-normalization", - "uuid", + "uuid 0.8.2", "zeroize", ] @@ -2458,7 +2705,7 @@ dependencies = [ "serde_repr", "tempfile", "tiny-bip39", - "uuid", + "uuid 0.8.2", ] [[package]] @@ -2602,7 +2849,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6085d7fd3cf84bd2b8fec150d54c8467fb491d8db9c460607c5534f653a0ee38" dependencies = [ - "darling", + "darling 0.13.4", "proc-macro2", "quote", "syn 1.0.109", @@ -2712,7 +2959,7 @@ dependencies = [ "getrandom", "hashers", "hex", - "http 0.2.11", + "http 0.2.12", "once_cell", "parking_lot 0.11.2", "pin-project", @@ -2858,9 +3105,9 @@ checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a" [[package]] name = "fastrand" -version = "2.0.1" +version = "2.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +checksum = "658bd65b1cf4c852a3cc96f18a8ce7b5640f6b703f905c7d74532294c2a63984" [[package]] name = "fastrlp" @@ -2901,9 +3148,9 @@ checksum = "ec54ac60a7f2ee9a97cad9946f9bf629a3bc6a7ae59e68983dc9318f5a54b81a" [[package]] name = "fiat-crypto" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1676f435fc1dadde4d03e43f5d62b259e1ce5f40bd4ffb21db2b42ebe59c1382" +checksum = "c007b1ae3abe1cb6f85a16305acd418b7ca6343b953633fee2b76d8f108b830f" [[package]] name = "field-offset" @@ -2915,6 +3162,12 @@ dependencies = [ "rustc_version 0.4.0", ] +[[package]] +name = "figlet-rs" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4742a071cd9694fc86f9fa1a08fa3e53d40cc899d7ee532295da2d085639fbc5" + [[package]] name = "filesystem" version = "0.1.0" @@ -3020,6 +3273,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "funty" version = "1.1.0" @@ -3093,9 +3352,9 @@ checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "445ba825b27408685aaecefd65178908c36c6e96aaf6d8599419d46e624192ba" +checksum = "52527eb5074e35e9339c6b4e8d12600c7128b68fb25dcb9fa9dec18f7c25f3a5" dependencies = [ "futures-core", "pin-project-lite", @@ -3109,7 +3368,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -3119,7 +3378,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35bd3cf68c183738046838e300353e4716c674dc5e56890de4826801a6622a28" dependencies = [ "futures-io", - "rustls", + "rustls 0.21.11", ] [[package]] @@ -3212,9 +3471,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.12" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "190092ea657667030ac6a35e305e62fc4dd69fd98ac98631e5d3a2b1575a12b5" +checksum = "94b22e06ecb0110981051723910cbf0b5f5e09a2062dd7663334ee79a9d1286c" dependencies = [ "cfg-if", "js-sys", @@ -3235,12 +3494,12 @@ dependencies = [ [[package]] name = "ghash" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d930750de5717d2dd0b8c0d42c076c0e884c81a73e6cab859bbd2339c71e3e40" +checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1" dependencies = [ "opaque-debug", - "polyval 0.6.1", + "polyval 0.6.2", ] [[package]] @@ -3266,7 +3525,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -3339,27 +3598,8 @@ dependencies = [ "futures-core", "futures-sink", "futures-util", - "http 0.2.11", - "indexmap 2.2.5", - "slab", - "tokio", - "tokio-util 0.7.10", - "tracing", -] - -[[package]] -name = "h2" -version = "0.4.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "816ec7294445779408f36fe57bc5b7fc1cf59664059096c65f905c1c61f58069" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http 1.0.0", - "indexmap 2.2.5", + "http 0.2.12", + "indexmap 2.2.6", "slab", "tokio", "tokio-util 0.7.10", @@ -3439,7 +3679,7 @@ dependencies = [ "base64 0.21.7", "bytes", "headers-core", - "http 0.2.11", + "http 0.2.12", "httpdate", "mime", "sha1", @@ -3451,7 +3691,7 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7f66481bfee273957b1f20485a4ff3362987f85b2c236580d81b4eb7a326429" dependencies = [ - "http 0.2.11", + "http 0.2.12", ] [[package]] @@ -3498,9 +3738,9 @@ checksum = "b07f60793ff0a4d9cef0f18e63b5357e06209987153a64648c972c1e5aff336f" [[package]] name = "hickory-proto" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "091a6fbccf4860009355e3efc52ff4acf37a63489aad7435372d44ceeb6fbbcf" +checksum = "07698b8420e2f0d6447a436ba999ec85d8fbf2a398bbd737b82cac4a2e96e512" dependencies = [ "async-trait", "cfg-if", @@ -3523,9 +3763,9 @@ dependencies = [ [[package]] name = "hickory-resolver" -version = "0.24.0" +version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35b8f021164e6a984c9030023544c57789c51760065cd510572fedcfb04164e8" +checksum = "28757f23aa75c98f254cf0405e6d8c25b831b32921b050a66692427679b1f243" dependencies = [ "cfg-if", "futures-util", @@ -3567,7 +3807,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" dependencies = [ - "crypto-mac 0.11.1", + "crypto-mac 0.11.0", "digest 0.9.0", ] @@ -3604,9 +3844,9 @@ dependencies = [ [[package]] name = "http" -version = "0.2.11" +version = "0.2.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8947b1a6fad4393052c7ba1f4cd97bed3e953a95c79c92ad9b051a04611d9fbb" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" dependencies = [ "bytes", "fnv", @@ -3615,9 +3855,9 @@ dependencies = [ [[package]] name = "http" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b32afd38673a8016f7c9ae69e5af41a58f81b1d31689040f2f1959594ce194ea" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" dependencies = [ "bytes", "fnv", @@ -3631,7 +3871,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2" dependencies = [ "bytes", - "http 0.2.11", + "http 0.2.12", "pin-project-lite", ] @@ -3642,18 +3882,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" dependencies = [ "bytes", - "http 1.0.0", + "http 1.1.0", ] [[package]] name = "http-body-util" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cb79eb393015dadd30fc252023adb0b2400a0caee0fa2a077e6e21a551e840" +checksum = "0475f8b2ac86659c21b64320d5d653f9efe42acd2a4e560073ec61a155a34f1d" dependencies = [ "bytes", - "futures-util", - "http 1.0.0", + "futures-core", + "http 1.1.0", "http-body 1.0.0", "pin-project-lite", ] @@ -3754,8 +3994,8 @@ dependencies = [ "futures-channel", "futures-core", "futures-util", - "h2 0.3.26", - "http 0.2.11", + "h2", + "http 0.2.12", "http-body 0.4.6", "httparse", "httpdate", @@ -3770,15 +4010,14 @@ dependencies = [ [[package]] name = "hyper" -version = "1.2.0" +version = "1.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186548d73ac615b32a73aafe38fb4f56c0d340e110e5a200bcadbaf2e199263a" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" dependencies = [ "bytes", "futures-channel", "futures-util", - "h2 0.4.4", - "http 1.0.0", + "http 1.1.0", "http-body 1.0.0", "httparse", "httpdate", @@ -3795,11 +4034,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ec3efd23720e2049821a693cbc7e65ea87c72f1c58ff2f9522ff332b1491e590" dependencies = [ "futures-util", - "http 0.2.11", + "http 0.2.12", "hyper 0.14.28", - "rustls", + "rustls 0.21.11", "tokio", - "tokio-rustls", + "tokio-rustls 0.24.1", ] [[package]] @@ -3823,9 +4062,9 @@ checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" dependencies = [ "bytes", "futures-util", - "http 1.0.0", + "http 1.1.0", "http-body 1.0.0", - "hyper 1.2.0", + "hyper 1.3.1", "pin-project-lite", "socket2 0.5.6", "tokio", @@ -3919,7 +4158,7 @@ dependencies = [ "attohttpc", "bytes", "futures", - "http 0.2.11", + "http 0.2.12", "hyper 0.14.28", "log", "rand", @@ -4002,9 +4241,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.5" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b0b929d511467233429c45a44ac1dcaa21ba0f5ba11e4879e6ed28ddb4f9df4" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -4037,7 +4276,7 @@ version = "0.2.0" dependencies = [ "bytes", "hex", - "yaml-rust", + "serde_yml", ] [[package]] @@ -4067,7 +4306,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" dependencies = [ "socket2 0.5.6", - "widestring 1.0.2", + "widestring 1.1.0", "windows-sys 0.48.0", "winreg", ] @@ -4100,9 +4339,9 @@ dependencies = [ [[package]] name = "itoa" -version = "1.0.10" +version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" +checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "jemalloc-ctl" @@ -4137,18 +4376,18 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.28" +version = "0.1.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" +checksum = "685a7d121ee3f65ae4fddd72b25a04bb36b6af81bc0828f7d5434c0fe60fa3a2" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" +checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d" dependencies = [ "wasm-bindgen", ] @@ -4261,7 +4500,7 @@ dependencies = [ "account_utils", "beacon_chain", "bls", - "clap", + "clap 2.34.0", "clap_utils", "deposit_contract", "directory", @@ -4349,12 +4588,12 @@ dependencies = [ [[package]] name = "libloading" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2caa5afb8bf9f3a2652760ce7d4f62d21c4d5a423e68466fca30df82f2330164" +checksum = "0c2a198fb6b0eada2a8df47933734e6d35d350665a33a3593d7164fa52c75c19" dependencies = [ "cfg-if", - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -4509,7 +4748,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "999ec70441b2fb35355076726a6bc466c932e9bdc66f6a11c6c0aa17c7ab9be0" dependencies = [ "asn1_der", - "bs58 0.5.0", + "bs58 0.5.1", "ed25519-dalek", "hkdf", "libsecp256k1", @@ -4640,7 +4879,7 @@ dependencies = [ "quinn", "rand", "ring 0.16.20", - "rustls", + "rustls 0.21.11", "socket2 0.5.6", "thiserror", "tokio", @@ -4679,7 +4918,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -4711,8 +4950,8 @@ dependencies = [ "libp2p-identity", "rcgen", "ring 0.16.20", - "rustls", - "rustls-webpki", + "rustls 0.21.11", + "rustls-webpki 0.101.7", "thiserror", "x509-parser", "yasna", @@ -4751,13 +4990,12 @@ dependencies = [ [[package]] name = "libredox" -version = "0.0.1" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85c833ca1e66078851dba29046874e38f08b2c883700aa29a03ddd3b23814ee8" +checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "libc", - "redox_syscall 0.4.1", ] [[package]] @@ -4821,9 +5059,9 @@ dependencies = [ [[package]] name = "libz-sys" -version = "1.1.15" +version = "1.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "037731f5d3aaa87a5675e895b63ddff1a87624bc29f77004ea829809654e48f6" +checksum = "5e143b5e666b2695d28f6bca6497720813f699c9602dd7f5cac91008b8ada7f9" dependencies = [ "cc", "pkg-config", @@ -4840,7 +5078,7 @@ dependencies = [ "beacon_processor", "bls", "boot_node", - "clap", + "clap 2.34.0", "clap_utils", "database_manager", "directory", @@ -5148,15 +5386,15 @@ checksum = "8878cd8d1b3c8c8ae4b2ba0a36652b7cf192f618a599a7fbdfa25cffd4ea72dd" [[package]] name = "memchr" -version = "2.7.1" +version = "2.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +checksum = "6c8640c5d730cb13ebd907d8d04b52f55ac9a2eec55b440c8892f40d56c76c1d" [[package]] name = "memoffset" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a634b1c61a95585bd15607c6ab0c4e5b226e695ff2800ba0cdccddf208c406c" +checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" dependencies = [ "autocfg", ] @@ -5188,7 +5426,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37cb4045d5677b7da537f8cb5d0730d5b6414e3cc81c61e4b50e1f0cbdc73909" dependencies = [ - "darling", + "darling 0.13.4", "itertools", "proc-macro2", "quote", @@ -5417,9 +5655,9 @@ dependencies = [ [[package]] name = "netlink-sys" -version = "0.8.5" +version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6471bf08e7ac0135876a9581bf3217ef0333c191c128d34878079f42ee150411" +checksum = "416060d346fbaf1f23f9512963e3e878f1a78e707cb699ba9215761754244307" dependencies = [ "bytes", "futures", @@ -5492,12 +5730,13 @@ dependencies = [ [[package]] name = "nix" -version = "0.27.1" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2eb04e9c688eff1c89d72b407f168cf79bb9e867a9d3323ed6c01519eb9cc053" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "cfg-if", + "cfg_aliases", "libc", ] @@ -5701,7 +5940,7 @@ version = "0.10.64" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a0481286a310808298130d22dd1fef0fa571e05a8f44ec801801e84b216b1f" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "cfg-if", "foreign-types", "libc", @@ -5718,7 +5957,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -5738,9 +5977,9 @@ dependencies = [ [[package]] name = "openssl-sys" -version = "0.9.101" +version = "0.9.102" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dda2b0f344e78efc2facf7d195d098df0dd72151b26ab98da807afc26c198dff" +checksum = "c597637d56fbc83893a35eb0dd04b2b8e7a50c91e64e9493e398b5df4fb45fa2" dependencies = [ "cc", "libc", @@ -5772,6 +6011,16 @@ dependencies = [ "types", ] +[[package]] +name = "os_pipe" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57119c3b893986491ec9aa85056780d3a0f3cf4da7cc09dd3650dbd6c6738fb9" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "overload" version = "0.1.1" @@ -5919,7 +6168,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d95f5254224e617595d2cc3cc73ff0a5eaf2637519e25f03388154e9378b6ffa" dependencies = [ - "crypto-mac 0.11.1", + "crypto-mac 0.11.0", ] [[package]] @@ -5951,11 +6200,11 @@ dependencies = [ [[package]] name = "pem" -version = "3.0.3" +version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8fcc794035347fb64beda2d3b462595dd2753e3f268d89c5aae77e8cf2c310" +checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae" dependencies = [ - "base64 0.21.7", + "base64 0.22.0", "serde", ] @@ -5976,9 +6225,9 @@ checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" [[package]] name = "pest" -version = "2.7.8" +version = "2.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f8023d0fb78c8e03784ea1c7f3fa36e68a723138990b8d5a47d916b651e7a8" +checksum = "311fb059dee1a7b802f036316d790138c613a4e8b180c822e3925a662e9f0c95" dependencies = [ "memchr", "thiserror", @@ -6015,29 +6264,29 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0302c4a0442c456bd56f841aee5c3bfd17967563f6fadc9ceb9f9c23cf3807e0" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.1.4" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] name = "pin-project-lite" -version = "0.2.13" +version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" [[package]] name = "pin-utils" @@ -6061,7 +6310,7 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" dependencies = [ - "der 0.7.8", + "der 0.7.9", "spki 0.7.3", ] @@ -6079,9 +6328,9 @@ checksum = "e8d0eef3571242013a0d5dc84861c3ae4a652e56e12adf8bdc26ff5f8cb34c94" [[package]] name = "platforms" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626dec3cac7cc0e1577a2ec3fc496277ec2baa084bebad95bb6fdbfae235f84c" +checksum = "db23d408679286588f4d4644f965003d056e3dd5abcaaa938116871d7ce2fee7" [[package]] name = "plotters" @@ -6113,14 +6362,15 @@ dependencies = [ [[package]] name = "polling" -version = "3.5.0" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24f040dee2588b4963afb4e420540439d126f73fdacf4a9c486a96d840bac3c9" +checksum = "e0c976a60b2d7e99d6f229e414670a9b85d13ac305cc6d1e9c134de58c5aaaf6" dependencies = [ "cfg-if", "concurrent-queue", + "hermit-abi 0.3.9", "pin-project-lite", - "rustix 0.38.31", + "rustix 0.38.33", "tracing", "windows-sys 0.52.0", ] @@ -6145,14 +6395,14 @@ dependencies = [ "cfg-if", "cpufeatures", "opaque-debug", - "universal-hash 0.4.1", + "universal-hash 0.4.0", ] [[package]] name = "polyval" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d52cff9d1d4dee5fe6d03729099f4a310a41179e0a10dbf542039873f2e826fb" +checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ "cfg-if", "cpufeatures", @@ -6275,9 +6525,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.78" +version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" +checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba" dependencies = [ "unicode-ident", ] @@ -6314,9 +6564,9 @@ dependencies = [ [[package]] name = "prometheus-client" -version = "0.22.1" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f87c10af16e0af74010d2a123d202e8363c04db5acfa91d8747f64a8524da3a" +checksum = "c1ca959da22a332509f2a73ae9e5f23f9dcfc31fd3a54d71f159495bd5909baa" dependencies = [ "dtoa", "itoa", @@ -6332,7 +6582,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -6343,13 +6593,13 @@ checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" dependencies = [ "bit-set", "bit-vec", - "bitflags 2.4.2", + "bitflags 2.5.0", "lazy_static", "num-traits", "rand", "rand_chacha", "rand_xorshift", - "regex-syntax 0.8.2", + "regex-syntax 0.8.3", "rusty-fork", "tempfile", "unarray", @@ -6468,7 +6718,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls", + "rustls 0.21.11", "thiserror", "tokio", "tracing", @@ -6484,7 +6734,7 @@ dependencies = [ "rand", "ring 0.16.20", "rustc-hash", - "rustls", + "rustls 0.21.11", "slab", "thiserror", "tinyvec", @@ -6506,9 +6756,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.35" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] @@ -6587,9 +6837,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4963ed1bc86e4f3ee217022bd855b297cef07fb9eac5dfa1f788b220b49b3bd" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" dependencies = [ "either", "rayon-core", @@ -6611,7 +6861,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52c4f3084aa3bc7dfbba4eff4fab2a54db4324965d8872ab933565e6fbd83bc6" dependencies = [ - "pem 3.0.3", + "pem 3.0.4", "ring 0.16.20", "time", "yasna", @@ -6637,9 +6887,9 @@ dependencies = [ [[package]] name = "redox_users" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a18479200779601e498ada4e8c1e1f50e3ee19deb0259c25825a98b5603b2cb4" +checksum = "bd283d9651eeda4b2a83a43c1c91b266c40fd76ecd39a50a8c630ae69dc72891" dependencies = [ "getrandom", "libredox", @@ -6648,14 +6898,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.3" +version = "1.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" +checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.5", - "regex-syntax 0.8.2", + "regex-automata 0.4.6", + "regex-syntax 0.8.3", ] [[package]] @@ -6669,13 +6919,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.5" +version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" +checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax 0.8.3", ] [[package]] @@ -6686,23 +6936,23 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" +checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" [[package]] name = "reqwest" -version = "0.11.24" +version = "0.11.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" +checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62" dependencies = [ "base64 0.21.7", "bytes", "encoding_rs", "futures-core", "futures-util", - "h2 0.3.26", - "http 0.2.11", + "h2", + "http 0.2.12", "http-body 0.4.6", "hyper 0.14.28", "hyper-rustls", @@ -6715,16 +6965,16 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls", - "rustls-pemfile", + "rustls 0.21.11", + "rustls-pemfile 1.0.4", "serde", "serde_json", "serde_urlencoded", - "sync_wrapper", + "sync_wrapper 0.1.2", "system-configuration", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.24.1", "tokio-util 0.7.10", "tower-service", "url", @@ -6803,6 +7053,30 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" +[[package]] +name = "rlg" +version = "0.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6ccf670238310d5c31a52fed1a3314620d037a64f1e5fbdc71b2c50909134dc" +dependencies = [ + "dtt", + "tokio", + "vrd 0.0.4", +] + +[[package]] +name = "rlg" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e02c717e23f67b23032a4acb01cf63534d6259938d592e6d2451c02f09fc368" +dependencies = [ + "dtt", + "hostname", + "serde_json", + "tokio", + "vrd 0.0.5", +] + [[package]] name = "rlp" version = "0.5.2" @@ -6851,9 +7125,9 @@ dependencies = [ [[package]] name = "ruint" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49b1d9521f889713d1221270fdd63370feca7e5c71a18745343402fa86e4f04f" +checksum = "8f308135fef9fc398342da5472ce7c484529df23743fb7c734e0f3d472971e62" dependencies = [ "alloy-rlp", "ark-ff 0.3.0", @@ -6954,11 +7228,11 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.31" +version = "0.38.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" +checksum = "e3cc72858054fcff6d7dea32df2aeaee6a7c24227366d7ea429aada2f26b16ad" dependencies = [ - "bitflags 2.4.2", + "bitflags 2.5.0", "errno", "libc", "linux-raw-sys 0.4.13", @@ -6967,16 +7241,30 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.10" +version = "0.21.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" +checksum = "7fecbfb7b1444f477b345853b1fce097a2c6fb637b2bfb87e6bc5db0f043fae4" dependencies = [ "log", "ring 0.17.8", - "rustls-webpki", + "rustls-webpki 0.101.7", "sct", ] +[[package]] +name = "rustls" +version = "0.22.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf4ef73721ac7bcd79b2b315da7779d8fc09718c6b3d2d1b2d94850eb8c18432" +dependencies = [ + "log", + "ring 0.17.8", + "rustls-pki-types", + "rustls-webpki 0.102.2", + "subtle", + "zeroize", +] + [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -6986,6 +7274,22 @@ dependencies = [ "base64 0.21.7", ] +[[package]] +name = "rustls-pemfile" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d" +dependencies = [ + "base64 0.22.0", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecd36cc4259e3e4514335c4a138c6b43171a8d61d8f5c9348f9fc7529416f247" + [[package]] name = "rustls-webpki" version = "0.101.7" @@ -6996,11 +7300,22 @@ dependencies = [ "untrusted 0.9.0", ] +[[package]] +name = "rustls-webpki" +version = "0.102.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faaa0a62740bedb9b2ef5afa303da42764c012f743917351dc9a237ea1663610" +dependencies = [ + "ring 0.17.8", + "rustls-pki-types", + "untrusted 0.9.0", +] + [[package]] name = "rustversion" -version = "1.0.14" +version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +checksum = "80af6f9131f277a45a3fba6ce8e2258037bb0477a67e610d3c1fe046ab31de47" [[package]] name = "rusty-fork" @@ -7055,9 +7370,9 @@ dependencies = [ [[package]] name = "scale-info" -version = "2.10.0" +version = "2.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7d66a1128282b7ef025a8ead62a4a9fcf017382ec53b8ffbf4d7bf77bd3c60" +checksum = "7c453e59a955f81fb62ee5d596b450383d699f152d350e9d23a0db2adb78e4c0" dependencies = [ "cfg-if", "derive_more", @@ -7067,9 +7382,9 @@ dependencies = [ [[package]] name = "scale-info-derive" -version = "2.10.0" +version = "2.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf2c68b89cafb3b8d918dd07b42be0da66ff202cf1155c5739a4e0c1ea0dc19" +checksum = "18cf6c6447f813ef19eb450e985bcce6705f9ce7660db221b59093d15c79c4b7" dependencies = [ "proc-macro-crate 1.3.1", "proc-macro2", @@ -7150,7 +7465,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct 0.2.0", - "der 0.7.8", + "der 0.7.9", "generic-array", "pkcs8 0.10.2", "subtle", @@ -7159,9 +7474,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.9.2" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05b64fb303737d99b81884b2c63433e9ae28abebe5eb5045dcdd175dc2ecf4de" +checksum = "770452e37cad93e0a50d5abc3990d2bc351c36d0328f86cefec2f2fb206eaef6" dependencies = [ "bitflags 1.3.2", "core-foundation", @@ -7172,9 +7487,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.9.1" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e932934257d3b408ed8f30db49d85ea163bfe74961f017f405b025af298f0c7a" +checksum = "41f3cc463c0ef97e11c3461a9d3787412d30e8e7eb907c79180c4a57bf7c04ef" dependencies = [ "core-foundation-sys", "libc", @@ -7223,9 +7538,9 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.197" +version = "1.0.198" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +checksum = "9846a40c979031340571da2545a4e5b7c4163bdae79b301d5f86d03979451fcc" dependencies = [ "serde_derive", ] @@ -7252,20 +7567,20 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.197" +version = "1.0.198" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +checksum = "e88edab869b01783ba905e7d0153f9fc1a6505a96e4ad3018011eedb838566d9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] name = "serde_json" -version = "1.0.114" +version = "1.0.116" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +checksum = "3e17db7126d17feb94eb3fad46bf1a96b034e8aacbc2e775fe81505f8b0b2813" dependencies = [ "itoa", "ryu", @@ -7274,9 +7589,9 @@ dependencies = [ [[package]] name = "serde_path_to_error" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd154a240de39fdebcf5775d2675c204d7c13cf39a4c697be6493c8e734337c" +checksum = "af99884400da37c88f5e9146b7f1fd0fbcae8f6eec4e9da38b67d05486f814a6" dependencies = [ "itoa", "serde", @@ -7284,13 +7599,13 @@ dependencies = [ [[package]] name = "serde_repr" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b2e6b945e9d3df726b65d6ee24060aff8e3533d431f677a9695db04eff9dfdb" +checksum = "6c64451ba24fc7a6a2d60fc75dd9c83c90903b19028d4eff35e88fc1e86564e9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -7330,7 +7645,7 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ - "darling", + "darling 0.13.4", "proc-macro2", "quote", "syn 1.0.109", @@ -7338,17 +7653,38 @@ dependencies = [ [[package]] name = "serde_yaml" -version = "0.9.32" +version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fd075d994154d4a774f95b51fb96bdc2832b0ea48425c92546073816cda1f2f" +checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.2.5", + "indexmap 2.2.6", "itoa", "ryu", "serde", "unsafe-libyaml", ] +[[package]] +name = "serde_yml" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "196c3750ff0411738366b0d9534ca55fc74d04a3d4284a450039de950bd11938" +dependencies = [ + "dtt", + "env_logger 0.11.3", + "figlet-rs", + "indexmap 2.2.6", + "itoa", + "log", + "openssl", + "rlg 0.0.3", + "ryu", + "serde", + "unsafe-libyaml", + "uuid 1.8.0", + "xtasks", +] + [[package]] name = "sha1" version = "0.10.6" @@ -7425,6 +7761,22 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "shared_child" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0d94659ad3c2137fef23ae75b03d5241d633f8acded53d672decfa0e6e0caef" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "shell-words" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" + [[package]] name = "shlex" version = "1.3.0" @@ -7433,9 +7785,9 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8229b473baa5980ac72ef434c4415e70c4b5e71b423043adb4ba059f89c99a1" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" dependencies = [ "libc", ] @@ -7476,7 +7828,7 @@ dependencies = [ name = "simulator" version = "0.2.0" dependencies = [ - "clap", + "clap 2.34.0", "env_logger 0.9.3", "eth1", "eth1_test_rig", @@ -7686,9 +8038,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.13.1" +version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" +checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" [[package]] name = "snap" @@ -7762,7 +8114,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" dependencies = [ "base64ct", - "der 0.7.8", + "der 0.7.9", ] [[package]] @@ -7875,6 +8227,12 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + [[package]] name = "strum" version = "0.24.1" @@ -7899,9 +8257,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.4.1" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "superstruct" @@ -7909,7 +8267,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75b9e5728aa1a87141cefd4e7509903fc01fa0dcb108022b1e841a67c5159fc5" dependencies = [ - "darling", + "darling 0.13.4", "itertools", "proc-macro2", "quote", @@ -7939,9 +8297,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.52" +version = "2.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b699d15b36d1f02c3e7c69f8ffef53de37aefae075d8488d4ba1a7788d574a07" +checksum = "909518bc7b1c9b779f1bbf07f2929d35af9f0f37e47c6e9ef7f9dddc1e1821f3" dependencies = [ "proc-macro2", "quote", @@ -7954,6 +8312,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" + [[package]] name = "synstructure" version = "0.12.6" @@ -8060,7 +8424,7 @@ checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1" dependencies = [ "cfg-if", "fastrand", - "rustix 0.38.31", + "rustix 0.38.33", "windows-sys 0.52.0", ] @@ -8128,22 +8492,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.57" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" +checksum = "f0126ad08bff79f29fc3ae6a55cc72352056dfff61e3ff8bb7129476d44b23aa" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.57" +version = "1.0.59" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" +checksum = "d1cd413b5d558b4c5bf3680e324a6fa5014e7b7c067a51e69dbdf47eb7148b66" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -8167,9 +8531,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.34" +version = "0.3.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" +checksum = "5dfd88e563464686c916c7e46e623e520ddc6d79fa6641390f2e3fa86e83e885" dependencies = [ "deranged", "itoa", @@ -8188,9 +8552,9 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" +checksum = "3f252a68540fde3a3877aeea552b832b40ab9a69e318efd078774a01ddee1ccf" dependencies = [ "num-conv", "time-core", @@ -8262,15 +8626,16 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.36.0" +version = "1.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" +checksum = "1adbebffeca75fcfd058afa480fb6c0b81e165a0323f9c9d39c9697e37c46787" dependencies = [ "backtrace", "bytes", "libc", "mio", "num_cpus", + "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", "socket2 0.5.6", @@ -8296,7 +8661,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -8341,15 +8706,26 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls", + "rustls 0.21.11", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" +dependencies = [ + "rustls 0.22.4", + "rustls-pki-types", "tokio", ] [[package]] name = "tokio-stream" -version = "0.1.14" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +checksum = "267ac89e0bec6e691e5813911606935d77c476ff49024f98abcea3e7b15e37af" dependencies = [ "futures-core", "pin-project-lite", @@ -8424,7 +8800,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.2.5", + "indexmap 2.2.6", "serde", "serde_spanned", "toml_datetime", @@ -8437,7 +8813,7 @@ version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" dependencies = [ - "indexmap 2.2.5", + "indexmap 2.2.6", "toml_datetime", "winnow", ] @@ -8502,7 +8878,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -8590,7 +8966,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "84303a9c7cda5f085a3ed9cd241d1e95e04d88aab1d679b02f212e653537ba86" dependencies = [ - "darling", + "darling 0.13.4", "quote", "syn 1.0.109", ] @@ -8744,9 +9120,9 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "universal-hash" -version = "0.4.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05" +checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" dependencies = [ "generic-array", "subtle", @@ -8764,9 +9140,9 @@ dependencies = [ [[package]] name = "unsafe-libyaml" -version = "0.2.10" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab4c90930b95a82d00dc9e9ac071b4991924390d46cbd0dfe566148667605e4b" +checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861" [[package]] name = "unsigned-varint" @@ -8826,6 +9202,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "uuid" version = "0.8.2" @@ -8836,6 +9218,15 @@ dependencies = [ "serde", ] +[[package]] +name = "uuid" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" +dependencies = [ + "getrandom", +] + [[package]] name = "validator_client" version = "0.3.5" @@ -8843,7 +9234,7 @@ dependencies = [ "account_utils", "bincode", "bls", - "clap", + "clap 2.34.0", "clap_utils", "deposit_contract", "directory", @@ -8855,7 +9246,7 @@ dependencies = [ "filesystem", "futures", "hex", - "hyper 1.2.0", + "hyper 1.3.1", "itertools", "lazy_static", "libsecp256k1", @@ -8915,7 +9306,7 @@ version = "0.1.0" dependencies = [ "account_utils", "bls", - "clap", + "clap 2.34.0", "clap_utils", "environment", "eth2", @@ -8964,6 +9355,25 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "vrd" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a81b8b5b404f3d7afa1b8142a6bc980c20cd68556c634c3db517871aa0402521" +dependencies = [ + "rand", +] + +[[package]] +name = "vrd" +version = "0.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee1067b8d17481f5be71b59d11c329e955ffe36348907e0a4a41b619682bb4af" +dependencies = [ + "rand", + "serde", +] + [[package]] name = "wait-timeout" version = "0.2.0" @@ -8994,29 +9404,28 @@ dependencies = [ [[package]] name = "warp" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e92e22e03ff1230c03a1a8ee37d2f89cd489e2e541b7550d6afad96faed169" +checksum = "4378d202ff965b011c64817db11d5829506d3404edeadb61f190d111da3f231c" dependencies = [ "bytes", "futures-channel", "futures-util", "headers", - "http 0.2.11", + "http 0.2.12", "hyper 0.14.28", "log", "mime", "mime_guess", "percent-encoding", "pin-project", - "rustls-pemfile", + "rustls-pemfile 2.1.2", "scoped-tls", "serde", "serde_json", "serde_urlencoded", "tokio", - "tokio-rustls", - "tokio-stream", + "tokio-rustls 0.25.0", "tokio-util 0.7.10", "tower-service", "tracing", @@ -9056,9 +9465,9 @@ checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" [[package]] name = "wasm-bindgen" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" +checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -9066,24 +9475,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" +checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" +checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0" dependencies = [ "cfg-if", "js-sys", @@ -9093,9 +9502,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" +checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -9103,22 +9512,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" +checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.91" +version = "0.2.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" +checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96" [[package]] name = "wasm-streams" @@ -9157,14 +9566,14 @@ dependencies = [ "beacon_node", "bls", "byteorder", - "clap", + "clap 2.34.0", "diesel", "diesel_migrations", "env_logger 0.9.3", "eth2", "hex", "http_api", - "hyper 1.2.0", + "hyper 1.3.1", "log", "logging", "network", @@ -9185,9 +9594,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.68" +version = "0.3.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" +checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef" dependencies = [ "js-sys", "wasm-bindgen", @@ -9227,9 +9636,9 @@ checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "whoami" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fec781d48b41f8163426ed18e8fc2864c12937df9ce54c88ede7bd47270893e" +checksum = "a44ab49fad634e88f55bf8f9bb3abd2f27d7204172a112c7c9987e01c1c94ea9" dependencies = [ "redox_syscall 0.4.1", "wasite", @@ -9244,9 +9653,9 @@ checksum = "c168940144dd21fd8046987c16a46a33d5fc84eec29ef9dcddc2ac9e31526b7c" [[package]] name = "widestring" -version = "1.0.2" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8" +checksum = "7219d36b6eac893fa81e84ebe06485e7dcbb616177469b142df14f1f4deb1311" [[package]] name = "winapi" @@ -9316,7 +9725,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -9343,7 +9752,7 @@ version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ - "windows-targets 0.52.4", + "windows-targets 0.52.5", ] [[package]] @@ -9378,17 +9787,18 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" dependencies = [ - "windows_aarch64_gnullvm 0.52.4", - "windows_aarch64_msvc 0.52.4", - "windows_i686_gnu 0.52.4", - "windows_i686_msvc 0.52.4", - "windows_x86_64_gnu 0.52.4", - "windows_x86_64_gnullvm 0.52.4", - "windows_x86_64_msvc 0.52.4", + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", ] [[package]] @@ -9405,9 +9815,9 @@ checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" [[package]] name = "windows_aarch64_msvc" @@ -9423,9 +9833,9 @@ checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" [[package]] name = "windows_i686_gnu" @@ -9441,9 +9851,15 @@ checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" [[package]] name = "windows_i686_msvc" @@ -9459,9 +9875,9 @@ checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" [[package]] name = "windows_x86_64_gnu" @@ -9477,9 +9893,9 @@ checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" [[package]] name = "windows_x86_64_gnullvm" @@ -9495,9 +9911,9 @@ checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" [[package]] name = "windows_x86_64_msvc" @@ -9513,9 +9929,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" -version = "0.52.4" +version = "0.52.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" [[package]] name = "winnow" @@ -9601,9 +10017,9 @@ dependencies = [ [[package]] name = "xml-rs" -version = "0.8.19" +version = "0.8.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fcb9cbac069e033553e8bb871be2fbdffcab578eb25bd0f7c508cedc6dcd75a" +checksum = "791978798f0597cfc70478424c2b4fdc2b7a8024aaff78497ef00f24ef674193" [[package]] name = "xmltree" @@ -9615,12 +10031,23 @@ dependencies = [ ] [[package]] -name = "yaml-rust" -version = "0.4.5" +name = "xtasks" +version = "0.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56c1936c4cc7a1c9ab21a1ebb602eb942ba868cbd44a99cb7cdc5892335e1c85" +checksum = "940db5674e301470e6cd91098b2c68a1fad751a1623575d1133f7456146e6d2f" dependencies = [ - "linked-hash-map", + "anyhow", + "clap 4.5.4", + "derive_builder", + "dialoguer", + "dtt", + "duct", + "fs_extra", + "glob", + "rlg 0.0.2", + "serde", + "serde_json", + "vrd 0.0.5", ] [[package]] @@ -9679,7 +10106,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -9699,7 +10126,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.52", + "syn 2.0.60", ] [[package]] @@ -9743,9 +10170,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.9+zstd.1.5.5" +version = "2.0.10+zstd.1.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e16efa8a874a0481a574084d34cc26fdb3b99627480f785888deb6386506656" +checksum = "c253a4914af5bafc8fa8c86ee400827e83cf6ec01195ec1f1ed8441bf00d65aa" dependencies = [ "cc", "pkg-config", diff --git a/Cargo.toml b/Cargo.toml index 24775b728de..69f6835f575 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -174,7 +174,7 @@ tree_hash = "0.5" tree_hash_derive = "0.5" url = "2" uuid = { version = "0.8", features = ["serde", "v4"] } -warp = { version = "0.3.6", default-features = false, features = ["tls"] } +warp = { version = "0.3.7", default-features = false, features = ["tls"] } zeroize = { version = "1", features = ["zeroize_derive"] } zip = "0.6" @@ -182,7 +182,7 @@ zip = "0.6" account_utils = { path = "common/account_utils" } beacon_chain = { path = "beacon_node/beacon_chain" } beacon_node = { path = "beacon_node" } -beacon_processor = { path = "beacon_node/beacon_processor" } +beacon_processor = { path = "beacon_node/beacon_processor" } bls = { path = "crypto/bls" } cached_tree_hash = { path = "consensus/cached_tree_hash" } clap_utils = { path = "common/clap_utils" } @@ -219,7 +219,7 @@ network = { path = "beacon_node/network" } operation_pool = { path = "beacon_node/operation_pool" } pretty_reqwest_error = { path = "common/pretty_reqwest_error" } proto_array = { path = "consensus/proto_array" } -safe_arith = {path = "consensus/safe_arith"} +safe_arith = { path = "consensus/safe_arith" } sensitive_url = { path = "common/sensitive_url" } slasher = { path = "slasher" } slashing_protection = { path = "validator_client/slashing_protection" } diff --git a/consensus/int_to_bytes/Cargo.toml b/consensus/int_to_bytes/Cargo.toml index 03bec9d3801..1958fac6e56 100644 --- a/consensus/int_to_bytes/Cargo.toml +++ b/consensus/int_to_bytes/Cargo.toml @@ -8,5 +8,5 @@ edition = { workspace = true } bytes = { workspace = true } [dev-dependencies] -yaml-rust = "0.4.4" hex = { workspace = true } +serde_yml = "0.0.4" diff --git a/consensus/int_to_bytes/src/lib.rs b/consensus/int_to_bytes/src/lib.rs index 589c72d249d..076e7154647 100644 --- a/consensus/int_to_bytes/src/lib.rs +++ b/consensus/int_to_bytes/src/lib.rs @@ -78,8 +78,7 @@ pub fn int_to_bytes96(int: u64) -> Vec { #[cfg(test)] mod tests { use super::*; - use std::{fs::File, io::prelude::*, path::PathBuf}; - use yaml_rust::yaml; + use std::{collections::HashMap, fs::File, io::prelude::*, path::PathBuf}; #[test] fn fixed_bytes32() { @@ -111,14 +110,13 @@ mod tests { file.read_to_string(&mut yaml_str).unwrap(); - let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); - let doc = &docs[0]; - let test_cases = doc["test_cases"].as_vec().unwrap(); + let docs: HashMap = serde_yml::from_str(&yaml_str).unwrap(); + let test_cases = docs["test_cases"].as_sequence().unwrap(); for test_case in test_cases { let byte_length = test_case["byte_length"].as_i64().unwrap() as u64; let int = test_case["int"].as_i64().unwrap() as u64; - let bytes_string = test_case["bytes"].clone().into_string().unwrap(); + let bytes_string = test_case["bytes"].as_str().unwrap(); let bytes = hex::decode(bytes_string.replace("0x", "")).unwrap(); match byte_length { From 67f8405921750f6a21e5c21d54688ecf31d4e825 Mon Sep 17 00:00:00 2001 From: Mac L Date: Tue, 23 Apr 2024 01:08:36 +1000 Subject: [PATCH 15/37] Update Simulator tests (#5520) * Rewrite Simulator * Add fallback simulator * Try Sean's test fix * More fixes * Cleanup * Merge branch 'unstable' into update-simulator * Update cli.rs * Add sync sim to basic sim * Formatting * Add fixes and new block production check * Merge branch 'unstable' of https://github.com/sigp/lighthouse into update-simulator * fix compile --- .github/workflows/test-suite.yml | 58 +-- Cargo.lock | 6 +- beacon_node/client/Cargo.toml | 1 + beacon_node/client/src/builder.rs | 16 + beacon_node/client/src/config.rs | 5 + .../test_utils/execution_block_generator.rs | 34 +- .../src/test_utils/handle_rpc.rs | 6 + .../execution_layer/src/test_utils/mod.rs | 1 + testing/simulator/Cargo.toml | 5 +- .../src/{eth1_sim.rs => basic_sim.rs} | 274 +++++------- testing/simulator/src/checks.rs | 174 +++++++- testing/simulator/src/cli.rs | 163 ++++---- testing/simulator/src/fallback_sim.rs | 261 ++++++++++++ testing/simulator/src/local_network.rs | 355 ++++++++++------ testing/simulator/src/main.rs | 26 +- testing/simulator/src/no_eth1_sim.rs | 172 -------- testing/simulator/src/sync_sim.rs | 390 ------------------ validator_client/src/block_service.rs | 1 + 18 files changed, 928 insertions(+), 1020 deletions(-) rename testing/simulator/src/{eth1_sim.rs => basic_sim.rs} (51%) create mode 100644 testing/simulator/src/fallback_sim.rs delete mode 100644 testing/simulator/src/no_eth1_sim.rs delete mode 100644 testing/simulator/src/sync_sim.rs diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index a8db6fab8fd..413dd2b95dd 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -225,8 +225,8 @@ jobs: run: docker build --build-arg FEATURES=portable -t lighthouse:local . - name: Test the built image run: docker run -t lighthouse:local lighthouse --version - eth1-simulator-ubuntu: - name: eth1-simulator-ubuntu + basic-simulator-ubuntu: + name: basic-simulator-ubuntu runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -235,42 +235,10 @@ jobs: with: channel: stable cache-target: release - - name: Install Foundry (anvil) - uses: foundry-rs/foundry-toolchain@v1 - with: - version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d - - name: Run the beacon chain sim that starts from an eth1 contract - run: cargo run --release --bin simulator eth1-sim - merge-transition-ubuntu: - name: merge-transition-ubuntu - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Get latest version of stable Rust - uses: moonrepo/setup-rust@v1 - with: - channel: stable - cache-target: release - - name: Install Foundry (anvil) - uses: foundry-rs/foundry-toolchain@v1 - with: - version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d - - name: Run the beacon chain sim and go through the merge transition - run: cargo run --release --bin simulator eth1-sim --post-merge - no-eth1-simulator-ubuntu: - name: no-eth1-simulator-ubuntu - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Get latest version of stable Rust - uses: moonrepo/setup-rust@v1 - with: - channel: stable - cache-target: release - - name: Run the beacon chain sim without an eth1 connection - run: cargo run --release --bin simulator no-eth1-sim - syncing-simulator-ubuntu: - name: syncing-simulator-ubuntu + - name: Run a basic beacon chain sim that starts from Bellatrix + run: cargo run --release --bin simulator basic-sim + fallback-simulator-ubuntu: + name: fallback-simulator-ubuntu runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -279,12 +247,8 @@ jobs: with: channel: stable cache-target: release - - name: Install Foundry (anvil) - uses: foundry-rs/foundry-toolchain@v1 - with: - version: nightly-ca67d15f4abd46394b324c50e21e66f306a1162d - - name: Run the syncing simulator - run: cargo run --release --bin simulator syncing-sim + - name: Run a beacon chain sim which tests VC fallback behaviour + run: cargo run --release --bin simulator fallback-sim doppelganger-protection-test: name: doppelganger-protection-test runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "CI", "small"]') || 'ubuntu-latest' }} @@ -442,10 +406,8 @@ jobs: 'state-transition-vectors-ubuntu', 'ef-tests-ubuntu', 'dockerfile-ubuntu', - 'eth1-simulator-ubuntu', - 'merge-transition-ubuntu', - 'no-eth1-simulator-ubuntu', - 'syncing-simulator-ubuntu', + 'basic-simulator-ubuntu', + 'fallback-simulator-ubuntu', 'doppelganger-protection-test', 'execution-engine-integration-ubuntu', 'check-code', diff --git a/Cargo.lock b/Cargo.lock index 79be0a55f18..4e3557c7efb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1420,6 +1420,7 @@ dependencies = [ "time", "timer", "tokio", + "tree_hash", "types", ] @@ -7831,13 +7832,16 @@ dependencies = [ "clap 2.34.0", "env_logger 0.9.3", "eth1", - "eth1_test_rig", + "eth2_network_config", + "ethereum-types 0.14.1", "execution_layer", "futures", "node_test_rig", "parking_lot 0.12.1", "rayon", "sensitive_url", + "serde_json", + "ssz_types", "tokio", "types", ] diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 03cbcc9ff7f..16c4a947a66 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -46,3 +46,4 @@ execution_layer = { workspace = true } beacon_processor = { workspace = true } num_cpus = { workspace = true } ethereum_ssz = { workspace = true } +tree_hash = { workspace = true } diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 6c505751542..e7f201b8521 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -26,6 +26,7 @@ use eth2::{ types::{BlockId, StateId}, BeaconNodeHttpClient, Error as ApiError, Timeouts, }; +use execution_layer::test_utils::generate_genesis_header; use execution_layer::ExecutionLayer; use futures::channel::mpsc::Receiver; use genesis::{interop_genesis_state, Eth1GenesisService, DEFAULT_ETH1_BLOCK_HASH}; @@ -267,6 +268,21 @@ where )?; builder.genesis_state(genesis_state).map(|v| (v, None))? } + ClientGenesis::InteropMerge { + validator_count, + genesis_time, + } => { + let execution_payload_header = generate_genesis_header(&spec, true); + let keypairs = generate_deterministic_keypairs(validator_count); + let genesis_state = interop_genesis_state( + &keypairs, + genesis_time, + Hash256::from_slice(DEFAULT_ETH1_BLOCK_HASH), + execution_payload_header, + &spec, + )?; + builder.genesis_state(genesis_state).map(|v| (v, None))? + } ClientGenesis::GenesisState => { info!( context.log(), diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index 48ad77abc58..a441e2c186c 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -24,6 +24,11 @@ pub enum ClientGenesis { validator_count: usize, genesis_time: u64, }, + // Creates a genesis state similar to the 2019 Canada specs, but starting post-Merge. + InteropMerge { + validator_count: usize, + genesis_time: u64, + }, /// Reads the genesis state and other persisted data from the `Store`. FromStore, /// Connects to an eth1 node and waits until it can create the genesis state from the deposit diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 08f8ca68435..bac2304fa85 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -91,7 +91,14 @@ impl Block { pub fn as_execution_block_with_tx(&self) -> Option> { match self { Block::PoS(payload) => Some(payload.clone().try_into().unwrap()), - Block::PoW(_) => None, + Block::PoW(block) => Some( + ExecutionPayload::Merge(ExecutionPayloadMerge { + block_hash: block.block_hash, + ..Default::default() + }) + .try_into() + .unwrap(), + ), } } } @@ -190,6 +197,19 @@ impl ExecutionBlockGenerator { .map(|block| block.as_execution_block(self.terminal_total_difficulty)) } + pub fn genesis_block(&self) -> Option> { + if let Some(genesis_block_hash) = self.block_hashes.get(&0) { + self.blocks.get(genesis_block_hash.first()?).cloned() + } else { + None + } + } + + pub fn genesis_execution_block(&self) -> Option { + self.genesis_block() + .map(|block| block.as_execution_block(self.terminal_total_difficulty)) + } + pub fn block_by_number(&self, number: u64) -> Option> { // Get the latest canonical head block let mut latest_block = self.latest_block()?; @@ -502,13 +522,6 @@ impl ExecutionBlockGenerator { let id = match payload_attributes { None => None, Some(attributes) => { - if !self.blocks.iter().any(|(_, block)| { - block.block_hash() == self.terminal_block_hash - || block.block_number() == self.terminal_block_number - }) { - return Err("refusing to create payload id before terminal block".to_string()); - } - let parent = self .blocks .get(&head_block_hash) @@ -766,12 +779,14 @@ pub fn generate_genesis_header( generate_genesis_block(spec.terminal_total_difficulty, DEFAULT_TERMINAL_BLOCK) .ok() .map(|block| block.block_hash); + let empty_transactions_root = Transactions::::empty().tree_hash_root(); match genesis_fork { ForkName::Base | ForkName::Altair => None, ForkName::Merge => { if post_transition_merge { let mut header = ExecutionPayloadHeader::Merge(<_>::default()); *header.block_hash_mut() = genesis_block_hash.unwrap_or_default(); + *header.transactions_root_mut() = empty_transactions_root; Some(header) } else { Some(ExecutionPayloadHeader::::Merge(<_>::default())) @@ -780,16 +795,19 @@ pub fn generate_genesis_header( ForkName::Capella => { let mut header = ExecutionPayloadHeader::Capella(<_>::default()); *header.block_hash_mut() = genesis_block_hash.unwrap_or_default(); + *header.transactions_root_mut() = empty_transactions_root; Some(header) } ForkName::Deneb => { let mut header = ExecutionPayloadHeader::Deneb(<_>::default()); *header.block_hash_mut() = genesis_block_hash.unwrap_or_default(); + *header.transactions_root_mut() = empty_transactions_root; Some(header) } ForkName::Electra => { let mut header = ExecutionPayloadHeader::Electra(<_>::default()); *header.block_hash_mut() = genesis_block_hash.unwrap_or_default(); + *header.transactions_root_mut() = empty_transactions_root; Some(header) } } diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index 77d972ab88e..e0ca07dcc6e 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -49,6 +49,12 @@ pub async fn handle_rpc( .latest_execution_block(), ) .unwrap()), + "0x0" => Ok(serde_json::to_value( + ctx.execution_block_generator + .read() + .genesis_execution_block(), + ) + .unwrap()), other => Err(( format!("The tag {} is not supported", other), BAD_PARAMS_ERROR_CODE, diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index eeb449b9920..43a6ee8ac22 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -58,6 +58,7 @@ mod mock_builder; mod mock_execution_layer; /// Configuration for the MockExecutionLayer. +#[derive(Clone)] pub struct MockExecutionConfig { pub server_config: Config, pub jwt_key: JwtKey, diff --git a/testing/simulator/Cargo.toml b/testing/simulator/Cargo.toml index eadcaf51b20..d7ff7b3dd85 100644 --- a/testing/simulator/Cargo.toml +++ b/testing/simulator/Cargo.toml @@ -14,8 +14,11 @@ types = { workspace = true } parking_lot = { workspace = true } futures = { workspace = true } tokio = { workspace = true } -eth1_test_rig = { workspace = true } env_logger = { workspace = true } clap = { workspace = true } rayon = { workspace = true } sensitive_url = { path = "../../common/sensitive_url" } +ssz_types = { workspace = true } +ethereum-types = { workspace = true } +eth2_network_config = { workspace = true } +serde_json = { workspace = true } diff --git a/testing/simulator/src/eth1_sim.rs b/testing/simulator/src/basic_sim.rs similarity index 51% rename from testing/simulator/src/eth1_sim.rs rename to testing/simulator/src/basic_sim.rs index 20c7c9ce9ab..755bb71b430 100644 --- a/testing/simulator/src/eth1_sim.rs +++ b/testing/simulator/src/basic_sim.rs @@ -1,50 +1,48 @@ -use crate::local_network::{EXECUTION_PORT, TERMINAL_BLOCK, TERMINAL_DIFFICULTY}; +use crate::local_network::LocalNetworkParams; +use crate::local_network::TERMINAL_BLOCK; use crate::{checks, LocalNetwork}; use clap::ArgMatches; -use eth1::{Eth1Endpoint, DEFAULT_CHAIN_ID}; -use eth1_test_rig::AnvilEth1Instance; use crate::retry::with_retry; -use execution_layer::http::deposit_methods::Eth1Id; use futures::prelude::*; -use node_test_rig::environment::RuntimeContext; use node_test_rig::{ environment::{EnvironmentBuilder, LoggerConfig}, - testing_client_config, testing_validator_config, ApiTopic, ClientConfig, ClientGenesis, - ValidatorFiles, + testing_validator_config, ApiTopic, ValidatorFiles, }; use rayon::prelude::*; -use sensitive_url::SensitiveUrl; use std::cmp::max; -use std::net::Ipv4Addr; use std::time::Duration; use tokio::time::sleep; use types::{Epoch, EthSpec, MinimalEthSpec}; const END_EPOCH: u64 = 16; -const ALTAIR_FORK_EPOCH: u64 = 1; -const BELLATRIX_FORK_EPOCH: u64 = 2; +const GENESIS_DELAY: u64 = 32; +const ALTAIR_FORK_EPOCH: u64 = 0; +const BELLATRIX_FORK_EPOCH: u64 = 0; +const CAPELLA_FORK_EPOCH: u64 = 1; +const DENEB_FORK_EPOCH: u64 = 2; +//const ELECTRA_FORK_EPOCH: u64 = 3; const SUGGESTED_FEE_RECIPIENT: [u8; 20] = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]; -pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { - let node_count = value_t!(matches, "nodes", usize).expect("missing nodes default"); - let proposer_nodes = value_t!(matches, "proposer-nodes", usize).unwrap_or(0); - println!("PROPOSER-NODES: {}", proposer_nodes); - let validators_per_node = value_t!(matches, "validators_per_node", usize) - .expect("missing validators_per_node default"); +pub fn run_basic_sim(matches: &ArgMatches) -> Result<(), String> { + let node_count = value_t!(matches, "nodes", usize).expect("Missing nodes default"); + let proposer_nodes = + value_t!(matches, "proposer-nodes", usize).expect("Missing proposer-nodes default"); + let validators_per_node = value_t!(matches, "validators-per-node", usize) + .expect("Missing validators-per-node default"); let speed_up_factor = - value_t!(matches, "speed_up_factor", u64).expect("missing speed_up_factor default"); - let continue_after_checks = matches.is_present("continue_after_checks"); - let post_merge_sim = matches.is_present("post-merge"); + value_t!(matches, "speed-up-factor", u64).expect("Missing speed-up-factor default"); + let log_level = value_t!(matches, "debug-level", String).expect("Missing default log-level"); + let continue_after_checks = matches.is_present("continue-after-checks"); - println!("Beacon Chain Simulator:"); - println!(" nodes:{}, proposer_nodes: {}", node_count, proposer_nodes); - - println!(" validators_per_node:{}", validators_per_node); - println!(" post merge simulation:{}", post_merge_sim); - println!(" continue_after_checks:{}", continue_after_checks); + println!("Basic Simulator:"); + println!(" nodes: {}", node_count); + println!(" proposer-nodes: {}", proposer_nodes); + println!(" validators-per-node: {}", validators_per_node); + println!(" speed-up-factor: {}", speed_up_factor); + println!(" continue-after-checks: {}", continue_after_checks); // Generate the directories and keystores required for the validator clients. let validator_files = (0..node_count) @@ -65,8 +63,8 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { let mut env = EnvironmentBuilder::minimal() .initialize_logger(LoggerConfig { path: None, - debug_level: String::from("debug"), - logfile_debug_level: String::from("debug"), + debug_level: log_level.clone(), + logfile_debug_level: log_level, log_format: None, logfile_format: None, log_color: false, @@ -80,32 +78,29 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { .multi_threaded_tokio_runtime()? .build()?; - let eth1_block_time = Duration::from_millis(15_000 / speed_up_factor); - let spec = &mut env.eth2_config.spec; let total_validator_count = validators_per_node * node_count; - let altair_fork_version = spec.altair_fork_version; - let bellatrix_fork_version = spec.bellatrix_fork_version; + let genesis_delay = GENESIS_DELAY; + + // Convenience variables. Update these values when adding a newer fork. + let latest_fork_version = spec.deneb_fork_version; + let latest_fork_start_epoch = DENEB_FORK_EPOCH; spec.seconds_per_slot /= speed_up_factor; spec.seconds_per_slot = max(1, spec.seconds_per_slot); - spec.eth1_follow_distance = 16; - spec.genesis_delay = eth1_block_time.as_secs() * spec.eth1_follow_distance * 2; + spec.genesis_delay = genesis_delay; spec.min_genesis_time = 0; spec.min_genesis_active_validator_count = total_validator_count as u64; - spec.seconds_per_eth1_block = eth1_block_time.as_secs(); spec.altair_fork_epoch = Some(Epoch::new(ALTAIR_FORK_EPOCH)); - // Set these parameters only if we are doing a merge simulation - if post_merge_sim { - spec.terminal_total_difficulty = TERMINAL_DIFFICULTY.into(); - spec.bellatrix_fork_epoch = Some(Epoch::new(BELLATRIX_FORK_EPOCH)); - } + spec.bellatrix_fork_epoch = Some(Epoch::new(BELLATRIX_FORK_EPOCH)); + spec.capella_fork_epoch = Some(Epoch::new(CAPELLA_FORK_EPOCH)); + spec.deneb_fork_epoch = Some(Epoch::new(DENEB_FORK_EPOCH)); + //spec.electra_fork_epoch = Some(Epoch::new(ELECTRA_FORK_EPOCH)); - let seconds_per_slot = spec.seconds_per_slot; let slot_duration = Duration::from_secs(spec.seconds_per_slot); + let slots_per_epoch = MinimalEthSpec::slots_per_epoch(); let initial_validator_count = spec.min_genesis_active_validator_count as usize; - let deposit_amount = env.eth2_config.spec.max_effective_balance; let context = env.core_context(); @@ -114,36 +109,36 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { * Create a new `LocalNetwork` with one beacon node. */ let max_retries = 3; - let (network, beacon_config) = with_retry(max_retries, || { - Box::pin(create_local_network( + let (network, beacon_config, mock_execution_config) = with_retry(max_retries, || { + Box::pin(LocalNetwork::create_local_network( + None, + None, LocalNetworkParams { - eth1_block_time, - total_validator_count, - deposit_amount, + validator_count: total_validator_count, node_count, proposer_nodes, - post_merge_sim, + genesis_delay, }, context.clone(), )) }) .await?; - /* - * One by one, add beacon nodes to the network. - */ - for _ in 0..node_count - 1 { + // Add nodes to the network. + for _ in 0..node_count { network - .add_beacon_node(beacon_config.clone(), false) + .add_beacon_node(beacon_config.clone(), mock_execution_config.clone(), false) .await?; } /* * One by one, add proposer nodes to the network. */ - for _ in 0..proposer_nodes - 1 { + for _ in 0..proposer_nodes { println!("Adding a proposer node"); - network.add_beacon_node(beacon_config.clone(), true).await?; + network + .add_beacon_node(beacon_config.clone(), mock_execution_config.clone(), true) + .await?; } /* @@ -156,9 +151,7 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { executor.spawn( async move { let mut validator_config = testing_validator_config(); - if post_merge_sim { - validator_config.fee_recipient = Some(SUGGESTED_FEE_RECIPIENT.into()); - } + validator_config.fee_recipient = Some(SUGGESTED_FEE_RECIPIENT.into()); println!("Adding validator client {}", i); // Enable broadcast on every 4th node. @@ -175,7 +168,7 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { .await } else { network_1 - .add_validator_client(validator_config, i, files, i % 2 == 0) + .add_validator_client(validator_config, i, files) .await } .expect("should add validator"); @@ -184,25 +177,15 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { ); } + // Set all payloads as valid. This effectively assumes the EL is infalliable. + network.execution_nodes.write().iter().for_each(|node| { + node.server.all_payloads_valid(); + }); + let duration_to_genesis = network.duration_to_genesis().await; println!("Duration to genesis: {}", duration_to_genesis.as_secs()); sleep(duration_to_genesis).await; - if post_merge_sim { - let executor = executor.clone(); - let network_2 = network.clone(); - executor.spawn( - async move { - println!("Mining pow blocks"); - let mut interval = tokio::time::interval(Duration::from_secs(seconds_per_slot)); - for i in 1..=TERMINAL_BLOCK + 1 { - interval.tick().await; - let _ = network_2.mine_pow_blocks(i); - } - }, - "pow_mining", - ); - } /* * Start the checks that ensure the network performs as expected. * @@ -211,6 +194,7 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { * tests start at the right time. Whilst this is works well for now, it's subject to * breakage by changes to the VC. */ + let network_1 = network.clone(); let ( finalization, @@ -221,13 +205,16 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { sync_aggregate, transition, light_client_update, + blobs, + start_node_with_delay, + sync, ) = futures::join!( // Check that the chain finalizes at the first given opportunity. checks::verify_first_finalization(network.clone(), slot_duration), // Check that a block is produced at every slot. checks::verify_full_block_production_up_to( network.clone(), - Epoch::new(END_EPOCH).start_slot(MinimalEthSpec::slots_per_epoch()), + Epoch::new(END_EPOCH).start_slot(slots_per_epoch), slot_duration, ), // Check that the chain starts with the expected validator count. @@ -246,41 +233,55 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { // Check that all nodes have transitioned to the required fork. checks::verify_fork_version( network.clone(), - if post_merge_sim { - Epoch::new(BELLATRIX_FORK_EPOCH) - } else { - Epoch::new(ALTAIR_FORK_EPOCH) - }, + Epoch::new(latest_fork_start_epoch), slot_duration, - if post_merge_sim { - bellatrix_fork_version - } else { - altair_fork_version - } + latest_fork_version, ), // Check that all sync aggregates are full. checks::verify_full_sync_aggregates_up_to( network.clone(), // Start checking for sync_aggregates at `FORK_EPOCH + 1` to account for // inefficiencies in finding subnet peers at the `fork_slot`. - Epoch::new(ALTAIR_FORK_EPOCH + 1).start_slot(MinimalEthSpec::slots_per_epoch()), - Epoch::new(END_EPOCH).start_slot(MinimalEthSpec::slots_per_epoch()), + Epoch::new(ALTAIR_FORK_EPOCH + 1).start_slot(slots_per_epoch), + Epoch::new(END_EPOCH).start_slot(slots_per_epoch), slot_duration, ), // Check that the transition block is finalized. checks::verify_transition_block_finalized( network.clone(), - Epoch::new(TERMINAL_BLOCK / MinimalEthSpec::slots_per_epoch()), + Epoch::new(TERMINAL_BLOCK / slots_per_epoch), slot_duration, - post_merge_sim + true, ), checks::verify_light_client_updates( network.clone(), // Sync aggregate available from slot 1 after Altair fork transition. - Epoch::new(ALTAIR_FORK_EPOCH).start_slot(MinimalEthSpec::slots_per_epoch()) + 1, - Epoch::new(END_EPOCH).start_slot(MinimalEthSpec::slots_per_epoch()), + Epoch::new(ALTAIR_FORK_EPOCH).start_slot(slots_per_epoch) + 1, + Epoch::new(END_EPOCH).start_slot(slots_per_epoch), slot_duration - ) + ), + checks::verify_full_blob_production_up_to( + network.clone(), + // Blobs should be available immediately after the Deneb fork. + Epoch::new(DENEB_FORK_EPOCH).start_slot(slots_per_epoch), + Epoch::new(END_EPOCH).start_slot(slots_per_epoch), + slot_duration + ), + network_1.add_beacon_node_with_delay( + beacon_config.clone(), + mock_execution_config.clone(), + END_EPOCH - 1, + slot_duration, + slots_per_epoch + ), + checks::ensure_node_synced_up_to_slot( + network.clone(), + // This must be set to be the node which was just created. Should be equal to + // `node_count`. + node_count, + Epoch::new(END_EPOCH).start_slot(slots_per_epoch), + slot_duration, + ), ); block_prod?; @@ -291,6 +292,9 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { sync_aggregate?; transition?; light_client_update?; + blobs?; + start_node_with_delay?; + sync?; // The `final_future` either completes immediately or never completes, depending on the value // of `continue_after_checks`. @@ -321,89 +325,3 @@ pub fn run_eth1_sim(matches: &ArgMatches) -> Result<(), String> { Ok(()) } - -struct LocalNetworkParams { - eth1_block_time: Duration, - total_validator_count: usize, - deposit_amount: u64, - node_count: usize, - proposer_nodes: usize, - post_merge_sim: bool, -} - -async fn create_local_network( - LocalNetworkParams { - eth1_block_time, - total_validator_count, - deposit_amount, - node_count, - proposer_nodes, - post_merge_sim, - }: LocalNetworkParams, - context: RuntimeContext, -) -> Result<(LocalNetwork, ClientConfig), String> { - /* - * Deploy the deposit contract, spawn tasks to keep creating new blocks and deposit - * validators. - */ - let anvil_eth1_instance = AnvilEth1Instance::new(DEFAULT_CHAIN_ID.into()).await?; - let deposit_contract = anvil_eth1_instance.deposit_contract; - let chain_id = anvil_eth1_instance.anvil.chain_id(); - let anvil = anvil_eth1_instance.anvil; - let eth1_endpoint = - SensitiveUrl::parse(anvil.endpoint().as_str()).expect("Unable to parse anvil endpoint."); - let deposit_contract_address = deposit_contract.address(); - - // Start a timer that produces eth1 blocks on an interval. - tokio::spawn(async move { - let mut interval = tokio::time::interval(eth1_block_time); - loop { - interval.tick().await; - let _ = anvil.evm_mine().await; - } - }); - - // Submit deposits to the deposit contract. - tokio::spawn(async move { - for i in 0..total_validator_count { - println!("Submitting deposit for validator {}...", i); - let _ = deposit_contract - .deposit_deterministic_async::(i, deposit_amount) - .await; - } - }); - - let mut beacon_config = testing_client_config(); - - beacon_config.genesis = ClientGenesis::DepositContract; - beacon_config.eth1.endpoint = Eth1Endpoint::NoAuth(eth1_endpoint); - beacon_config.eth1.deposit_contract_address = deposit_contract_address; - beacon_config.eth1.deposit_contract_deploy_block = 0; - beacon_config.eth1.lowest_cached_block_number = 0; - beacon_config.eth1.follow_distance = 1; - beacon_config.eth1.node_far_behind_seconds = 20; - beacon_config.dummy_eth1_backend = false; - beacon_config.sync_eth1_chain = true; - beacon_config.eth1.auto_update_interval_millis = eth1_block_time.as_millis() as u64; - beacon_config.eth1.chain_id = Eth1Id::from(chain_id); - beacon_config.network.target_peers = node_count + proposer_nodes - 1; - - beacon_config.network.enr_address = (Some(Ipv4Addr::LOCALHOST), None); - beacon_config.network.enable_light_client_server = true; - beacon_config.chain.enable_light_client_server = true; - beacon_config.http_api.enable_light_client_server = true; - - if post_merge_sim { - let el_config = execution_layer::Config { - execution_endpoint: Some( - SensitiveUrl::parse(&format!("http://localhost:{}", EXECUTION_PORT)).unwrap(), - ), - ..Default::default() - }; - - beacon_config.execution_layer = Some(el_config); - } - - let network = LocalNetwork::new(context, beacon_config.clone()).await?; - Ok((network, beacon_config)) -} diff --git a/testing/simulator/src/checks.rs b/testing/simulator/src/checks.rs index d30e44a1174..03cc17fab3e 100644 --- a/testing/simulator/src/checks.rs +++ b/testing/simulator/src/checks.rs @@ -1,7 +1,7 @@ use crate::local_network::LocalNetwork; use node_test_rig::eth2::types::{BlockId, FinalityCheckpointsData, StateId}; use std::time::Duration; -use types::{Epoch, EthSpec, ExecPayload, ExecutionBlockHash, Hash256, Slot, Unsigned}; +use types::{Epoch, EthSpec, ExecPayload, ExecutionBlockHash, Slot, Unsigned}; /// Checks that all of the validators have on-boarded by the start of the second eth1 voting /// period. @@ -234,7 +234,7 @@ pub async fn verify_transition_block_finalized( } let first = block_hashes[0]; - if first.into_root() != Hash256::zero() && block_hashes.iter().all(|&item| item == first) { + if block_hashes.iter().all(|&item| item == first) { Ok(()) } else { Err(format!( @@ -333,3 +333,173 @@ pub(crate) async fn verify_light_client_updates( Ok(()) } + +/// Checks that a node is synced with the network. +/// Useful for ensuring that a node which started after genesis is able to sync to the head. +pub async fn ensure_node_synced_up_to_slot( + network: LocalNetwork, + node_index: usize, + upto_slot: Slot, + slot_duration: Duration, +) -> Result<(), String> { + slot_delay(upto_slot, slot_duration).await; + let node = &network + .remote_nodes()? + .get(node_index) + .expect("Should get node") + .clone(); + + let head = node + .get_beacon_blocks::(BlockId::Head) + .await + .ok() + .flatten() + .ok_or(format!("No head block exists on node {node_index}"))? + .data; + + // Check the head block is synced with the rest of the network. + if head.slot() >= upto_slot { + Ok(()) + } else { + Err(format!( + "Head not synced for node {node_index}. Found {}; Should be {upto_slot}", + head.slot() + )) + } +} + +/// Verifies that there's been blobs produced at every slot with a block from `blob_start_slot` up +/// to and including `upto_slot`. +pub async fn verify_full_blob_production_up_to( + network: LocalNetwork, + blob_start_slot: Slot, + upto_slot: Slot, + slot_duration: Duration, +) -> Result<(), String> { + slot_delay(upto_slot, slot_duration).await; + let remote_nodes = network.remote_nodes()?; + let remote_node = remote_nodes.first().unwrap(); + + for slot in blob_start_slot.as_u64()..=upto_slot.as_u64() { + // Ensure block exists. + let block = remote_node + .get_beacon_blocks::(BlockId::Slot(Slot::new(slot))) + .await + .ok() + .flatten(); + + // Only check blobs if the block exists. If you also want to ensure full block production, use + // the `verify_full_block_production_up_to` function. + if block.is_some() { + remote_node + .get_blobs::(BlockId::Slot(Slot::new(slot)), None) + .await + .map_err(|e| format!("Failed to get blobs at slot {slot:?}: {e:?}"))? + .ok_or_else(|| format!("No blobs available at slot {slot:?}"))?; + } + } + + Ok(()) +} + +// Causes the beacon node at `node_index` to disconnect from the execution layer. +pub async fn disconnect_from_execution_layer( + network: LocalNetwork, + node_index: usize, +) -> Result<(), String> { + eprintln!("Disabling Execution Node {node_index}"); + + // Force the execution node to return the `syncing` status. + network.execution_nodes.read()[node_index] + .server + .all_payloads_syncing(false); + Ok(()) +} + +// Causes the beacon node at `node_index` to reconnect from the execution layer. +pub async fn reconnect_to_execution_layer( + network: LocalNetwork, + node_index: usize, +) -> Result<(), String> { + network.execution_nodes.read()[node_index] + .server + .all_payloads_valid(); + + eprintln!("Enabling Execution Node {node_index}"); + Ok(()) +} + +/// Ensure all validators have attested correctly. +pub async fn check_attestation_correctness( + network: LocalNetwork, + start_epoch: u64, + upto_epoch: u64, + slot_duration: Duration, + // Select which node to query. Will use this node to determine the global network performance. + node_index: usize, + acceptable_attestation_performance: f64, +) -> Result<(), String> { + epoch_delay(Epoch::new(upto_epoch), slot_duration, E::slots_per_epoch()).await; + + let remote_node = &network.remote_nodes()?[node_index]; + + let results = remote_node + .get_lighthouse_analysis_attestation_performance( + Epoch::new(start_epoch), + Epoch::new(upto_epoch - 2), + "global".to_string(), + ) + .await + .map_err(|e| format!("Unable to get attestation performance: {e}"))?; + + let mut active_successes: f64 = 0.0; + let mut head_successes: f64 = 0.0; + let mut target_successes: f64 = 0.0; + let mut source_successes: f64 = 0.0; + + let mut total: f64 = 0.0; + + for result in results { + for epochs in result.epochs.values() { + total += 1.0; + + if epochs.active { + active_successes += 1.0; + } + if epochs.head { + head_successes += 1.0; + } + if epochs.target { + target_successes += 1.0; + } + if epochs.source { + source_successes += 1.0; + } + } + } + let active_percent = active_successes / total * 100.0; + let head_percent = head_successes / total * 100.0; + let target_percent = target_successes / total * 100.0; + let source_percent = source_successes / total * 100.0; + + eprintln!("Total Attestations: {}", total); + eprintln!("Active: {}: {}%", active_successes, active_percent); + eprintln!("Head: {}: {}%", head_successes, head_percent); + eprintln!("Target: {}: {}%", target_successes, target_percent); + eprintln!("Source: {}: {}%", source_successes, source_percent); + + if active_percent < acceptable_attestation_performance { + return Err("Active percent was below required level".to_string()); + } + if head_percent < acceptable_attestation_performance { + return Err("Head percent was below required level".to_string()); + } + if target_percent < acceptable_attestation_performance { + return Err("Target percent was below required level".to_string()); + } + if source_percent < acceptable_attestation_performance { + return Err("Source percent was below required level".to_string()); + } + + Ok(()) +} diff --git a/testing/simulator/src/cli.rs b/testing/simulator/src/cli.rs index ff80201051f..00af7e560ce 100644 --- a/testing/simulator/src/cli.rs +++ b/testing/simulator/src/cli.rs @@ -6,120 +6,121 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .author("Sigma Prime ") .about("Options for interacting with simulator") .subcommand( - SubCommand::with_name("eth1-sim") - .about( - "Lighthouse Beacon Chain Simulator creates `n` beacon node and validator clients, \ - each with `v` validators. A deposit contract is deployed at the start of the \ - simulation using a local `anvil` instance (you must have `anvil` \ - installed and avaliable on your path). All beacon nodes independently listen \ - for genesis from the deposit contract, then start operating. \ - \ + SubCommand::with_name("basic-sim") + .about( + "Runs a Beacon Chain simulation with `n` beacon node and validator clients, \ + each with `v` validators. \ + The simulation runs with a post-Merge Genesis using `mock-el`. \ As the simulation runs, there are checks made to ensure that all components \ are running correctly. If any of these checks fail, the simulation will \ exit immediately.", - ) - .arg(Arg::with_name("nodes") + ) + .arg( + Arg::with_name("nodes") .short("n") .long("nodes") .takes_value(true) - .default_value("4") - .help("Number of beacon nodes")) - .arg(Arg::with_name("proposer-nodes") + .default_value("3") + .help("Number of beacon nodes"), + ) + .arg( + Arg::with_name("proposer-nodes") .short("p") - .long("proposer_nodes") + .long("proposer-nodes") .takes_value(true) - .default_value("2") - .help("Number of proposer-only beacon nodes")) - .arg(Arg::with_name("validators_per_node") + .default_value("3") + .help("Number of proposer-only beacon nodes"), + ) + .arg( + Arg::with_name("validators-per-node") .short("v") - .long("validators_per_node") + .long("validators-per-node") .takes_value(true) .default_value("20") - .help("Number of validators")) - .arg(Arg::with_name("speed_up_factor") + .help("Number of validators"), + ) + .arg( + Arg::with_name("speed-up-factor") .short("s") - .long("speed_up_factor") + .long("speed-up-factor") .takes_value(true) .default_value("3") - .help("Speed up factor. Please use a divisor of 12.")) - .arg(Arg::with_name("post-merge") - .short("m") - .long("post-merge") - .takes_value(false) - .help("Simulate the merge transition")) - .arg(Arg::with_name("continue_after_checks") + .help("Speed up factor. Please use a divisor of 12."), + ) + .arg( + Arg::with_name("debug-level") + .short("d") + .long("debug-level") + .takes_value(true) + .default_value("debug") + .help("Set the severity level of the logs."), + ) + .arg( + Arg::with_name("continue-after-checks") .short("c") .long("continue_after_checks") .takes_value(false) - .help("Continue after checks (default false)")) + .help("Continue after checks (default false)"), + ), ) .subcommand( - SubCommand::with_name("no-eth1-sim") - .about("Runs a simulator that bypasses the eth1 chain. Useful for faster testing of - components that don't rely upon eth1") - .arg(Arg::with_name("nodes") - .short("n") - .long("nodes") + SubCommand::with_name("fallback-sim") + .about( + "Runs a Beacon Chain simulation with `c` validator clients where each VC is \ + connected to `b` beacon nodes with `v` validators. \ + During the simulation, all but the last connected BN for each VC are \ + disconnected from the execution layer, which causes the VC to fallback to the \ + single remaining BN. \ + At the end of the simulation, there are checks made to ensure that all VCs \ + efficiently performed this fallback, within a certain tolerance. \ + Otherwise, the simulation will exit and an error will be reported.", + ) + .arg( + Arg::with_name("vc-count") + .short("c") + .long("vc-count") .takes_value(true) - .default_value("4") - .help("Number of beacon nodes")) - .arg(Arg::with_name("proposer-nodes") - .short("p") - .long("proposer_nodes") + .default_value("3") + .help("Number of validator clients."), + ) + .arg( + Arg::with_name("bns-per-vc") + .short("b") + .long("bns-per-vc") .takes_value(true) .default_value("2") - .help("Number of proposer-only beacon nodes")) - .arg(Arg::with_name("validators_per_node") + .help("Number of beacon nodes per validator client."), + ) + .arg( + Arg::with_name("validators-per-vc") .short("v") - .long("validators_per_node") + .long("validators-per-vc") .takes_value(true) .default_value("20") - .help("Number of validators")) - .arg(Arg::with_name("speed_up_factor") - .short("s") - .long("speed_up_factor") - .takes_value(true) - .default_value("3") - .help("Speed up factor")) - .arg(Arg::with_name("continue_after_checks") - .short("c") - .long("continue_after_checks") - .takes_value(false) - .help("Continue after checks (default false)")) - ) - .subcommand( - SubCommand::with_name("syncing-sim") - .about("Run the syncing simulation") - .arg( - Arg::with_name("speedup") - .short("s") - .long("speedup") - .takes_value(true) - .default_value("15") - .help("Speed up factor for eth1 blocks and slot production"), + .help("Number of validators per client."), ) .arg( - Arg::with_name("initial_delay") - .short("i") - .long("initial_delay") + Arg::with_name("speed-up-factor") + .short("s") + .long("speed-up-factor") .takes_value(true) - .default_value("5") - .help("Epoch delay for new beacon node to start syncing"), + .default_value("3") + .help("Speed up factor. Please use a divisor of 12."), ) .arg( - Arg::with_name("sync_timeout") - .long("sync_timeout") + Arg::with_name("debug-level") + .short("d") + .long("debug-level") .takes_value(true) - .default_value("10") - .help("Number of epochs after which newly added beacon nodes must be synced"), + .default_value("debug") + .help("Set the severity level of the logs."), ) .arg( - Arg::with_name("strategy") - .long("strategy") - .takes_value(true) - .default_value("all") - .possible_values(&["one-node", "two-nodes", "mixed", "all"]) - .help("Sync verification strategy to run."), + Arg::with_name("continue-after-checks") + .short("c") + .long("continue_after_checks") + .takes_value(false) + .help("Continue after checks (default false)"), ), ) } diff --git a/testing/simulator/src/fallback_sim.rs b/testing/simulator/src/fallback_sim.rs new file mode 100644 index 00000000000..c9deeba04d9 --- /dev/null +++ b/testing/simulator/src/fallback_sim.rs @@ -0,0 +1,261 @@ +use crate::local_network::LocalNetworkParams; +use crate::{checks, LocalNetwork}; +use clap::ArgMatches; + +use crate::retry::with_retry; +use futures::prelude::*; +use node_test_rig::{ + environment::{EnvironmentBuilder, LoggerConfig}, + testing_validator_config, ValidatorFiles, +}; +use rayon::prelude::*; +use std::cmp::max; +use std::time::Duration; +use tokio::time::sleep; +use types::{Epoch, EthSpec, MinimalEthSpec}; + +const END_EPOCH: u64 = 16; +const GENESIS_DELAY: u64 = 32; +const ALTAIR_FORK_EPOCH: u64 = 0; +const BELLATRIX_FORK_EPOCH: u64 = 0; +const CAPELLA_FORK_EPOCH: u64 = 1; +const DENEB_FORK_EPOCH: u64 = 2; +//const ELECTRA_FORK_EPOCH: u64 = 3; + +// Since simulator tests are non-deterministic and there is a non-zero chance of missed +// attestations, define an acceptable network-wide attestation performance. +// +// This has potential to block CI so it should be set conservatively enough that spurious failures +// don't become very common, but not so conservatively that regressions to the fallback mechanism +// cannot be detected. +const ACCEPTABLE_FALLBACK_ATTESTATION_HIT_PERCENTAGE: f64 = 85.0; + +const SUGGESTED_FEE_RECIPIENT: [u8; 20] = + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1]; + +pub fn run_fallback_sim(matches: &ArgMatches) -> Result<(), String> { + let vc_count = value_t!(matches, "vc-count", usize).expect("Missing validator-count default"); + let validators_per_vc = + value_t!(matches, "validators-per-vc", usize).expect("Missing validators-per-vc default"); + let bns_per_vc = value_t!(matches, "bns-per-vc", usize).expect("Missing bns-per-vc default"); + assert!(bns_per_vc > 1); + let speed_up_factor = + value_t!(matches, "speed-up-factor", u64).expect("Missing speed-up-factor default"); + let log_level = value_t!(matches, "debug-level", String).expect("Missing default log-level"); + let continue_after_checks = matches.is_present("continue-after-checks"); + + println!("Fallback Simulator:"); + println!(" vc-count: {}", vc_count); + println!(" validators-per-vc: {}", validators_per_vc); + println!(" bns-per-vc: {}", bns_per_vc); + println!(" speed-up-factor: {}", speed_up_factor); + println!(" continue-after-checks: {}", continue_after_checks); + + // Generate the directories and keystores required for the validator clients. + let validator_files = (0..vc_count) + .into_par_iter() + .map(|i| { + println!( + "Generating keystores for validator {} of {}", + i + 1, + vc_count + ); + + let indices = (i * validators_per_vc..(i + 1) * validators_per_vc).collect::>(); + ValidatorFiles::with_keystores(&indices).unwrap() + }) + .collect::>(); + + let mut env = EnvironmentBuilder::minimal() + .initialize_logger(LoggerConfig { + path: None, + debug_level: log_level.clone(), + logfile_debug_level: log_level, + log_format: None, + logfile_format: None, + log_color: false, + disable_log_timestamp: false, + max_log_size: 0, + max_log_number: 0, + compression: false, + is_restricted: true, + sse_logging: false, + })? + .multi_threaded_tokio_runtime()? + .build()?; + + let spec = &mut env.eth2_config.spec; + + let total_validator_count = validators_per_vc * vc_count; + let node_count = vc_count * bns_per_vc; + + let genesis_delay = GENESIS_DELAY; + + spec.seconds_per_slot /= speed_up_factor; + spec.seconds_per_slot = max(1, spec.seconds_per_slot); + spec.genesis_delay = genesis_delay; + spec.min_genesis_time = 0; + spec.min_genesis_active_validator_count = total_validator_count as u64; + spec.altair_fork_epoch = Some(Epoch::new(ALTAIR_FORK_EPOCH)); + spec.bellatrix_fork_epoch = Some(Epoch::new(BELLATRIX_FORK_EPOCH)); + spec.capella_fork_epoch = Some(Epoch::new(CAPELLA_FORK_EPOCH)); + spec.deneb_fork_epoch = Some(Epoch::new(DENEB_FORK_EPOCH)); + //spec.electra_fork_epoch = Some(Epoch::new(ELECTRA_FORK_EPOCH)); + + let slot_duration = Duration::from_secs(spec.seconds_per_slot); + let slots_per_epoch = MinimalEthSpec::slots_per_epoch(); + + let disconnection_epoch = 1; + let epochs_disconnected = 14; + + let context = env.core_context(); + + let main_future = async { + /* + * Create a new `LocalNetwork` with one beacon node. + */ + let max_retries = 3; + let (network, beacon_config, mock_execution_config) = with_retry(max_retries, || { + Box::pin(LocalNetwork::create_local_network( + None, + None, + LocalNetworkParams { + validator_count: total_validator_count, + node_count, + proposer_nodes: 0, + genesis_delay, + }, + context.clone(), + )) + }) + .await?; + + // Add nodes to the network. + for _ in 0..node_count { + network + .add_beacon_node(beacon_config.clone(), mock_execution_config.clone(), false) + .await?; + } + + /* + * One by one, add validators to the network. + */ + let executor = context.executor.clone(); + for (i, files) in validator_files.into_iter().enumerate() { + let network_1 = network.clone(); + + let mut beacon_nodes = Vec::with_capacity(vc_count * bns_per_vc); + // Each VC gets a unique set of BNs which are not shared with any other VC. + for j in 0..bns_per_vc { + beacon_nodes.push(bns_per_vc * i + j) + } + + executor.spawn( + async move { + let mut validator_config = testing_validator_config(); + validator_config.fee_recipient = Some(SUGGESTED_FEE_RECIPIENT.into()); + println!("Adding validator client {}", i); + network_1 + .add_validator_client_with_fallbacks( + validator_config, + i, + beacon_nodes, + files, + ) + .await + .expect("should add validator"); + }, + "vc", + ); + } + + let duration_to_genesis = network.duration_to_genesis().await; + println!("Duration to genesis: {}", duration_to_genesis.as_secs()); + sleep(duration_to_genesis).await; + + let test_sequence = async { + checks::epoch_delay( + Epoch::new(disconnection_epoch), + slot_duration, + slots_per_epoch, + ) + .await; + // Iterate through each VC and disconnect all BNs but the last node for each VC. + for i in 0..vc_count { + for j in 0..(bns_per_vc - 1) { + let node_index = bns_per_vc * i + j; + checks::disconnect_from_execution_layer(network.clone(), node_index).await?; + } + } + checks::epoch_delay( + Epoch::new(epochs_disconnected), + slot_duration, + slots_per_epoch, + ) + .await; + // Enable all BNs. + for i in 0..node_count { + checks::reconnect_to_execution_layer(network.clone(), i).await?; + } + Ok::<(), String>(()) + }; + + /* + * Start the checks that ensure the network performs as expected. + * + * We start these checks immediately after the validators have started. This means we're + * relying on the validator futures to all return immediately after genesis so that these + * tests start at the right time. Whilst this is works well for now, it's subject to + * breakage by changes to the VC. + */ + + let (sequence, check_attestations, block_production) = futures::join!( + test_sequence, + checks::check_attestation_correctness( + network.clone(), + 0, + END_EPOCH, + slot_duration, + // Use the last node index as this will never have been disabled. + node_count - 1, + ACCEPTABLE_FALLBACK_ATTESTATION_HIT_PERCENTAGE, + ), + checks::verify_full_block_production_up_to( + network.clone(), + Epoch::new(END_EPOCH).start_slot(slots_per_epoch), + slot_duration, + ), + ); + sequence?; + block_production?; + check_attestations?; + + // The `final_future` either completes immediately or never completes, depending on the value + // of `continue_after_checks`. + + if continue_after_checks { + future::pending::<()>().await; + } + /* + * End the simulation by dropping the network. This will kill all running beacon nodes and + * validator clients. + */ + println!( + "Simulation complete. Finished with {} beacon nodes and {} validator clients", + network.beacon_node_count(), + network.validator_client_count() + ); + + // Be explicit about dropping the network, as this kills all the nodes. This ensures + // all the checks have adequate time to pass. + drop(network); + Ok::<(), String>(()) + }; + + env.runtime().block_on(main_future).unwrap(); + + env.fire_signal(); + env.shutdown_on_idle(); + + Ok(()) +} diff --git a/testing/simulator/src/local_network.rs b/testing/simulator/src/local_network.rs index 018954a5d3b..63f2ec93537 100644 --- a/testing/simulator/src/local_network.rs +++ b/testing/simulator/src/local_network.rs @@ -1,26 +1,95 @@ +use crate::checks::epoch_delay; +use eth2_network_config::TRUSTED_SETUP_BYTES; use node_test_rig::{ environment::RuntimeContext, eth2::{types::StateId, BeaconNodeHttpClient}, - ClientConfig, LocalBeaconNode, LocalExecutionNode, LocalValidatorClient, MockExecutionConfig, - MockServerConfig, ValidatorConfig, ValidatorFiles, + testing_client_config, ClientConfig, ClientGenesis, LocalBeaconNode, LocalExecutionNode, + LocalValidatorClient, MockExecutionConfig, MockServerConfig, ValidatorConfig, ValidatorFiles, }; use parking_lot::RwLock; use sensitive_url::SensitiveUrl; use std::{ + net::Ipv4Addr, ops::Deref, - time::{SystemTime, UNIX_EPOCH}, + sync::Arc, + time::{Duration, SystemTime, UNIX_EPOCH}, }; -use std::{sync::Arc, time::Duration}; -use types::{Epoch, EthSpec}; +use types::{ChainSpec, Epoch, EthSpec}; const BOOTNODE_PORT: u16 = 42424; const QUIC_PORT: u16 = 43424; -pub const INVALID_ADDRESS: &str = "http://127.0.0.1:42423"; pub const EXECUTION_PORT: u16 = 4000; -pub const TERMINAL_DIFFICULTY: u64 = 6400; -pub const TERMINAL_BLOCK: u64 = 64; +pub const TERMINAL_BLOCK: u64 = 0; + +pub struct LocalNetworkParams { + pub validator_count: usize, + pub node_count: usize, + pub proposer_nodes: usize, + pub genesis_delay: u64, +} + +fn default_client_config(network_params: LocalNetworkParams, genesis_time: u64) -> ClientConfig { + let mut beacon_config = testing_client_config(); + + beacon_config.genesis = ClientGenesis::InteropMerge { + validator_count: network_params.validator_count, + genesis_time, + }; + beacon_config.network.target_peers = + network_params.node_count + network_params.proposer_nodes - 1; + beacon_config.network.enr_address = (Some(Ipv4Addr::LOCALHOST), None); + beacon_config.network.enable_light_client_server = true; + beacon_config.network.discv5_config.enable_packet_filter = false; + beacon_config.chain.enable_light_client_server = true; + beacon_config.http_api.enable_light_client_server = true; + beacon_config.chain.optimistic_finalized_sync = false; + beacon_config.trusted_setup = + serde_json::from_reader(TRUSTED_SETUP_BYTES).expect("Trusted setup bytes should be valid"); + + let el_config = execution_layer::Config { + execution_endpoint: Some( + SensitiveUrl::parse(&format!("http://localhost:{}", EXECUTION_PORT)).unwrap(), + ), + ..Default::default() + }; + beacon_config.execution_layer = Some(el_config); + beacon_config +} + +fn default_mock_execution_config( + spec: &ChainSpec, + genesis_time: u64, +) -> MockExecutionConfig { + let mut mock_execution_config = MockExecutionConfig { + server_config: MockServerConfig { + listen_port: EXECUTION_PORT, + ..Default::default() + }, + ..Default::default() + }; + + if let Some(capella_fork_epoch) = spec.capella_fork_epoch { + mock_execution_config.shanghai_time = Some( + genesis_time + + spec.seconds_per_slot * E::slots_per_epoch() * capella_fork_epoch.as_u64(), + ) + } + if let Some(deneb_fork_epoch) = spec.deneb_fork_epoch { + mock_execution_config.cancun_time = Some( + genesis_time + spec.seconds_per_slot * E::slots_per_epoch() * deneb_fork_epoch.as_u64(), + ) + } + if let Some(electra_fork_epoch) = spec.electra_fork_epoch { + mock_execution_config.prague_time = Some( + genesis_time + + spec.seconds_per_slot * E::slots_per_epoch() * electra_fork_epoch.as_u64(), + ) + } + + mock_execution_config +} /// Helper struct to reduce `Arc` usage. pub struct Inner { @@ -55,56 +124,41 @@ impl Deref for LocalNetwork { } impl LocalNetwork { - /// Creates a new network with a single `BeaconNode` and a connected `ExecutionNode`. - pub async fn new( + pub async fn create_local_network( + client_config: Option, + mock_execution_config: Option, + network_params: LocalNetworkParams, context: RuntimeContext, - mut beacon_config: ClientConfig, - ) -> Result { - beacon_config.network.set_ipv4_listening_address( - std::net::Ipv4Addr::UNSPECIFIED, - BOOTNODE_PORT, - BOOTNODE_PORT, - QUIC_PORT, - ); - beacon_config.network.enr_udp4_port = Some(BOOTNODE_PORT.try_into().expect("non zero")); - beacon_config.network.enr_tcp4_port = Some(BOOTNODE_PORT.try_into().expect("non zero")); - beacon_config.network.discv5_config.table_filter = |_| true; + ) -> Result<(LocalNetwork, ClientConfig, MockExecutionConfig), String> { + let genesis_time: u64 = (SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(|_| "should get system time")? + + Duration::from_secs(network_params.genesis_delay)) + .as_secs(); - let execution_node = if let Some(el_config) = &mut beacon_config.execution_layer { - let mock_execution_config = MockExecutionConfig { - server_config: MockServerConfig { - listen_port: EXECUTION_PORT, - ..Default::default() - }, - terminal_block: TERMINAL_BLOCK, - terminal_difficulty: TERMINAL_DIFFICULTY.into(), - ..Default::default() - }; - let execution_node = LocalExecutionNode::new( - context.service_context("boot_node_el".into()), - mock_execution_config, - ); - el_config.default_datadir = execution_node.datadir.path().to_path_buf(); - el_config.secret_file = Some(execution_node.datadir.path().join("jwt.hex")); - el_config.execution_endpoint = - Some(SensitiveUrl::parse(&execution_node.server.url()).unwrap()); - vec![execution_node] + let beacon_config = if let Some(config) = client_config { + config } else { - vec![] + default_client_config(network_params, genesis_time) }; - let beacon_node = - LocalBeaconNode::production(context.service_context("boot_node".into()), beacon_config) - .await?; - Ok(Self { + let execution_config = if let Some(config) = mock_execution_config { + config + } else { + default_mock_execution_config::(&context.eth2_config().spec, genesis_time) + }; + + let network = Self { inner: Arc::new(Inner { context, - beacon_nodes: RwLock::new(vec![beacon_node]), + beacon_nodes: RwLock::new(vec![]), proposer_nodes: RwLock::new(vec![]), - execution_nodes: RwLock::new(execution_node), + execution_nodes: RwLock::new(vec![]), validator_clients: RwLock::new(vec![]), }), - }) + }; + + Ok((network, beacon_config, execution_config)) } /// Returns the number of beacon nodes in the network. @@ -131,77 +185,151 @@ impl LocalNetwork { self.validator_clients.read().len() } - /// Adds a beacon node to the network, connecting to the 0'th beacon node via ENR. - pub async fn add_beacon_node( + async fn construct_boot_node( + &self, + mut beacon_config: ClientConfig, + mock_execution_config: MockExecutionConfig, + ) -> Result<(LocalBeaconNode, LocalExecutionNode), String> { + beacon_config.network.set_ipv4_listening_address( + std::net::Ipv4Addr::UNSPECIFIED, + BOOTNODE_PORT, + BOOTNODE_PORT, + QUIC_PORT, + ); + + beacon_config.network.enr_udp4_port = Some(BOOTNODE_PORT.try_into().expect("non zero")); + beacon_config.network.enr_tcp4_port = Some(BOOTNODE_PORT.try_into().expect("non zero")); + beacon_config.network.discv5_config.table_filter = |_| true; + + let execution_node = LocalExecutionNode::new( + self.context.service_context("boot_node_el".into()), + mock_execution_config, + ); + + beacon_config.execution_layer = Some(execution_layer::Config { + execution_endpoint: Some(SensitiveUrl::parse(&execution_node.server.url()).unwrap()), + default_datadir: execution_node.datadir.path().to_path_buf(), + secret_file: Some(execution_node.datadir.path().join("jwt.hex")), + ..Default::default() + }); + + let beacon_node = LocalBeaconNode::production( + self.context.service_context("boot_node".into()), + beacon_config, + ) + .await?; + + Ok((beacon_node, execution_node)) + } + + async fn construct_beacon_node( &self, mut beacon_config: ClientConfig, + mut mock_execution_config: MockExecutionConfig, is_proposer: bool, - ) -> Result<(), String> { - let self_1 = self.clone(); - let count = self.beacon_node_count() as u16; - println!("Adding beacon node.."); - { - let read_lock = self.beacon_nodes.read(); + ) -> Result<(LocalBeaconNode, LocalExecutionNode), String> { + let count = (self.beacon_node_count() + self.proposer_node_count()) as u16; - let boot_node = read_lock.first().expect("should have at least one node"); + // Set config. + let libp2p_tcp_port = BOOTNODE_PORT + count; + let discv5_port = BOOTNODE_PORT + count; + beacon_config.network.set_ipv4_listening_address( + std::net::Ipv4Addr::UNSPECIFIED, + libp2p_tcp_port, + discv5_port, + QUIC_PORT + count, + ); + beacon_config.network.enr_udp4_port = Some(discv5_port.try_into().unwrap()); + beacon_config.network.enr_tcp4_port = Some(libp2p_tcp_port.try_into().unwrap()); + beacon_config.network.discv5_config.table_filter = |_| true; + beacon_config.network.proposer_only = is_proposer; - beacon_config.network.boot_nodes_enr.push( - boot_node - .client - .enr() - .expect("bootnode must have a network"), - ); - let count = (self.beacon_node_count() + self.proposer_node_count()) as u16; - let libp2p_tcp_port = BOOTNODE_PORT + count; - let discv5_port = BOOTNODE_PORT + count; - beacon_config.network.set_ipv4_listening_address( - std::net::Ipv4Addr::UNSPECIFIED, - libp2p_tcp_port, - discv5_port, - QUIC_PORT + count, - ); - beacon_config.network.enr_udp4_port = Some(discv5_port.try_into().unwrap()); - beacon_config.network.enr_tcp4_port = Some(libp2p_tcp_port.try_into().unwrap()); - beacon_config.network.discv5_config.table_filter = |_| true; - beacon_config.network.proposer_only = is_proposer; - } - if let Some(el_config) = &mut beacon_config.execution_layer { - let config = MockExecutionConfig { - server_config: MockServerConfig { - listen_port: EXECUTION_PORT + count, - ..Default::default() - }, - terminal_block: TERMINAL_BLOCK, - terminal_difficulty: TERMINAL_DIFFICULTY.into(), - ..Default::default() - }; - let execution_node = LocalExecutionNode::new( - self.context.service_context(format!("node_{}_el", count)), - config, - ); - el_config.default_datadir = execution_node.datadir.path().to_path_buf(); - el_config.secret_file = Some(execution_node.datadir.path().join("jwt.hex")); - el_config.execution_endpoint = - Some(SensitiveUrl::parse(&execution_node.server.url()).unwrap()); - self.execution_nodes.write().push(execution_node); - } + mock_execution_config.server_config.listen_port = EXECUTION_PORT + count; - // We create the beacon node without holding the lock, so that the lock isn't held - // across the await. This is only correct if this function never runs in parallel - // with itself (which at the time of writing, it does not). + // Construct execution node. + let execution_node = LocalExecutionNode::new( + self.context.service_context(format!("node_{}_el", count)), + mock_execution_config, + ); + + // Pair the beacon node and execution node. + beacon_config.execution_layer = Some(execution_layer::Config { + execution_endpoint: Some(SensitiveUrl::parse(&execution_node.server.url()).unwrap()), + default_datadir: execution_node.datadir.path().to_path_buf(), + secret_file: Some(execution_node.datadir.path().join("jwt.hex")), + ..Default::default() + }); + + // Construct beacon node using the config, let beacon_node = LocalBeaconNode::production( self.context.service_context(format!("node_{}", count)), beacon_config, ) .await?; + + Ok((beacon_node, execution_node)) + } + + /// Adds a beacon node to the network, connecting to the 0'th beacon node via ENR. + pub async fn add_beacon_node( + &self, + mut beacon_config: ClientConfig, + mock_execution_config: MockExecutionConfig, + is_proposer: bool, + ) -> Result<(), String> { + let first_bn_exists: bool; + { + let read_lock = self.beacon_nodes.read(); + let boot_node = read_lock.first(); + first_bn_exists = boot_node.is_some(); + + if let Some(boot_node) = boot_node { + // Modify beacon_config to add boot node details. + beacon_config.network.boot_nodes_enr.push( + boot_node + .client + .enr() + .expect("Bootnode must have a network."), + ); + } + } + let (beacon_node, execution_node) = if first_bn_exists { + // Network already exists. We construct a new node. + self.construct_beacon_node(beacon_config, mock_execution_config, is_proposer) + .await? + } else { + // Network does not exist. We construct a boot node. + self.construct_boot_node(beacon_config, mock_execution_config) + .await? + }; + // Add nodes to the network. + self.execution_nodes.write().push(execution_node); if is_proposer { - self_1.proposer_nodes.write().push(beacon_node); + self.proposer_nodes.write().push(beacon_node); } else { - self_1.beacon_nodes.write().push(beacon_node); + self.beacon_nodes.write().push(beacon_node); } Ok(()) } + // Add a new node with a delay. This node will not have validators and is only used to test + // sync. + pub async fn add_beacon_node_with_delay( + &self, + beacon_config: ClientConfig, + mock_execution_config: MockExecutionConfig, + wait_until_epoch: u64, + slot_duration: Duration, + slots_per_epoch: u64, + ) -> Result<(), String> { + epoch_delay(Epoch::new(wait_until_epoch), slot_duration, slots_per_epoch).await; + + self.add_beacon_node(beacon_config, mock_execution_config, false) + .await?; + + Ok(()) + } + /// Adds a validator client to the network, connecting it to the beacon node with index /// `beacon_node`. pub async fn add_validator_client( @@ -209,7 +337,6 @@ impl LocalNetwork { mut validator_config: ValidatorConfig, beacon_node: usize, validator_files: ValidatorFiles, - invalid_first_beacon_node: bool, //to test beacon node fallbacks ) -> Result<(), String> { let context = self .context @@ -240,11 +367,7 @@ impl LocalNetwork { format!("http://{}:{}", socket_addr.ip(), socket_addr.port()).as_str(), ) .unwrap(); - validator_config.beacon_nodes = if invalid_first_beacon_node { - vec![SensitiveUrl::parse(INVALID_ADDRESS).unwrap(), beacon_node] - } else { - vec![beacon_node] - }; + validator_config.beacon_nodes = vec![beacon_node]; // If we have a proposer node established, use it. if let Some(proposer_socket_addr) = proposer_socket_addr { @@ -293,11 +416,11 @@ impl LocalNetwork { .http_api_listen_addr() .expect("Must have http started") }; - let beacon_node = SensitiveUrl::parse( + let beacon_node_url = SensitiveUrl::parse( format!("http://{}:{}", socket_addr.ip(), socket_addr.port()).as_str(), ) .unwrap(); - beacon_node_urls.push(beacon_node); + beacon_node_urls.push(beacon_node_url); } validator_config.beacon_nodes = beacon_node_urls; @@ -325,7 +448,7 @@ impl LocalNetwork { } /// Return current epoch of bootnode. - pub async fn bootnode_epoch(&self) -> Result { + pub async fn _bootnode_epoch(&self) -> Result { let nodes = self.remote_nodes().expect("Failed to get remote nodes"); let bootnode = nodes.first().expect("Should contain bootnode"); bootnode @@ -335,16 +458,6 @@ impl LocalNetwork { .map(|body| body.unwrap().data.finalized.epoch) } - pub fn mine_pow_blocks(&self, block_number: u64) -> Result<(), String> { - let execution_nodes = self.execution_nodes.read(); - for execution_node in execution_nodes.iter() { - let mut block_gen = execution_node.server.ctx.execution_block_generator.write(); - block_gen.insert_pow_block(block_number)?; - println!("Mined pow block {}", block_number); - } - Ok(()) - } - pub async fn duration_to_genesis(&self) -> Duration { let nodes = self.remote_nodes().expect("Failed to get remote nodes"); let bootnode = nodes.first().expect("Should contain bootnode"); diff --git a/testing/simulator/src/main.rs b/testing/simulator/src/main.rs index e8af9c18067..d1a2d0dc672 100644 --- a/testing/simulator/src/main.rs +++ b/testing/simulator/src/main.rs @@ -1,10 +1,8 @@ -//! This crate provides a simluation that creates `n` beacon node and validator clients, each with -//! `v` validators. A deposit contract is deployed at the start of the simulation using a local -//! `anvil` instance (you must have `anvil` installed and avaliable on your path). All -//! beacon nodes independently listen for genesis from the deposit contract, then start operating. +//! This crate provides various simulations that create both beacon nodes and validator clients, +//! each with `v` validators. //! -//! As the simulation runs, there are checks made to ensure that all components are running -//! correctly. If any of these checks fail, the simulation will exit immediately. +//! When a simulation runs, there are checks made to ensure that all components are operating +//! as expected. If any of these checks fail, the simulation will exit immediately. //! //! ## Future works //! @@ -16,13 +14,12 @@ #[macro_use] extern crate clap; +mod basic_sim; mod checks; mod cli; -mod eth1_sim; +mod fallback_sim; mod local_network; -mod no_eth1_sim; mod retry; -mod sync_sim; use cli::cli_app; use env_logger::{Builder, Env}; @@ -37,21 +34,14 @@ fn main() { let matches = cli_app().get_matches(); match matches.subcommand() { - ("eth1-sim", Some(matches)) => match eth1_sim::run_eth1_sim(matches) { + ("basic-sim", Some(matches)) => match basic_sim::run_basic_sim(matches) { Ok(()) => println!("Simulation exited successfully"), Err(e) => { eprintln!("Simulation exited with error: {}", e); std::process::exit(1) } }, - ("no-eth1-sim", Some(matches)) => match no_eth1_sim::run_no_eth1_sim(matches) { - Ok(()) => println!("Simulation exited successfully"), - Err(e) => { - eprintln!("Simulation exited with error: {}", e); - std::process::exit(1) - } - }, - ("syncing-sim", Some(matches)) => match sync_sim::run_syncing_sim(matches) { + ("fallback-sim", Some(matches)) => match fallback_sim::run_fallback_sim(matches) { Ok(()) => println!("Simulation exited successfully"), Err(e) => { eprintln!("Simulation exited with error: {}", e); diff --git a/testing/simulator/src/no_eth1_sim.rs b/testing/simulator/src/no_eth1_sim.rs deleted file mode 100644 index fc18b1cd489..00000000000 --- a/testing/simulator/src/no_eth1_sim.rs +++ /dev/null @@ -1,172 +0,0 @@ -use crate::{checks, LocalNetwork}; -use clap::ArgMatches; -use futures::prelude::*; -use node_test_rig::{ - environment::{EnvironmentBuilder, LoggerConfig}, - testing_client_config, testing_validator_config, ClientGenesis, ValidatorFiles, -}; -use rayon::prelude::*; -use std::cmp::max; -use std::net::Ipv4Addr; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use tokio::time::sleep; -use types::{Epoch, EthSpec, MainnetEthSpec}; - -pub fn run_no_eth1_sim(matches: &ArgMatches) -> Result<(), String> { - let node_count = value_t!(matches, "nodes", usize).expect("missing nodes default"); - let validators_per_node = value_t!(matches, "validators_per_node", usize) - .expect("missing validators_per_node default"); - let speed_up_factor = - value_t!(matches, "speed_up_factor", u64).expect("missing speed_up_factor default"); - let continue_after_checks = matches.is_present("continue_after_checks"); - - println!("Beacon Chain Simulator:"); - println!(" nodes:{}", node_count); - println!(" validators_per_node:{}", validators_per_node); - println!(" continue_after_checks:{}", continue_after_checks); - - // Generate the directories and keystores required for the validator clients. - let validator_files = (0..node_count) - .into_par_iter() - .map(|i| { - println!( - "Generating keystores for validator {} of {}", - i + 1, - node_count - ); - - let indices = - (i * validators_per_node..(i + 1) * validators_per_node).collect::>(); - ValidatorFiles::with_keystores(&indices).unwrap() - }) - .collect::>(); - - let mut env = EnvironmentBuilder::mainnet() - .initialize_logger(LoggerConfig { - path: None, - debug_level: String::from("debug"), - logfile_debug_level: String::from("debug"), - log_format: None, - logfile_format: None, - log_color: false, - disable_log_timestamp: false, - max_log_size: 0, - max_log_number: 0, - compression: false, - is_restricted: true, - sse_logging: false, - })? - .multi_threaded_tokio_runtime()? - .build()?; - - let eth1_block_time = Duration::from_millis(15_000 / speed_up_factor); - - let spec = &mut env.eth2_config.spec; - - let total_validator_count = validators_per_node * node_count; - - spec.seconds_per_slot /= speed_up_factor; - spec.seconds_per_slot = max(1, spec.seconds_per_slot); - spec.eth1_follow_distance = 16; - spec.genesis_delay = eth1_block_time.as_secs() * spec.eth1_follow_distance * 2; - spec.min_genesis_time = 0; - spec.min_genesis_active_validator_count = total_validator_count as u64; - spec.seconds_per_eth1_block = 1; - - let genesis_delay = Duration::from_secs(5); - let genesis_time = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|_| "should get system time")? - + genesis_delay; - - let slot_duration = Duration::from_secs(spec.seconds_per_slot); - - let context = env.core_context(); - - let mut beacon_config = testing_client_config(); - - beacon_config.genesis = ClientGenesis::Interop { - validator_count: total_validator_count, - genesis_time: genesis_time.as_secs(), - }; - beacon_config.dummy_eth1_backend = true; - beacon_config.sync_eth1_chain = true; - - beacon_config.network.enr_address = (Some(Ipv4Addr::LOCALHOST), None); - - let main_future = async { - let network = LocalNetwork::new(context.clone(), beacon_config.clone()).await?; - /* - * One by one, add beacon nodes to the network. - */ - - for _ in 0..node_count - 1 { - network - .add_beacon_node(beacon_config.clone(), false) - .await?; - } - - /* - * Create a future that will add validator clients to the network. Each validator client is - * attached to a single corresponding beacon node. Spawn each validator in a new task. - */ - let executor = context.executor.clone(); - for (i, files) in validator_files.into_iter().enumerate() { - let network_1 = network.clone(); - executor.spawn( - async move { - println!("Adding validator client {}", i); - network_1 - .add_validator_client(testing_validator_config(), i, files, i % 2 == 0) - .await - .expect("should add validator"); - }, - "vc", - ); - } - - let duration_to_genesis = network.duration_to_genesis().await; - println!("Duration to genesis: {}", duration_to_genesis.as_secs()); - sleep(duration_to_genesis).await; - - let (finalization, block_prod) = futures::join!( - // Check that the chain finalizes at the first given opportunity. - checks::verify_first_finalization(network.clone(), slot_duration), - // Check that a block is produced at every slot. - checks::verify_full_block_production_up_to( - network.clone(), - Epoch::new(4).start_slot(MainnetEthSpec::slots_per_epoch()), - slot_duration, - ), - ); - finalization?; - block_prod?; - - // The `final_future` either completes immediately or never completes, depending on the value - // of `continue_after_checks`. - - if continue_after_checks { - future::pending::<()>().await; - } - /* - * End the simulation by dropping the network. This will kill all running beacon nodes and - * validator clients. - */ - println!( - "Simulation complete. Finished with {} beacon nodes and {} validator clients", - network.beacon_node_count() + network.proposer_node_count(), - network.validator_client_count() - ); - - // Be explicit about dropping the network, as this kills all the nodes. This ensures - // all the checks have adequate time to pass. - drop(network); - Ok::<(), String>(()) - }; - - env.runtime().block_on(main_future).unwrap(); - - env.fire_signal(); - env.shutdown_on_idle(); - Ok(()) -} diff --git a/testing/simulator/src/sync_sim.rs b/testing/simulator/src/sync_sim.rs deleted file mode 100644 index ba4ea4530af..00000000000 --- a/testing/simulator/src/sync_sim.rs +++ /dev/null @@ -1,390 +0,0 @@ -use crate::checks::{epoch_delay, verify_all_finalized_at}; -use crate::local_network::LocalNetwork; -use clap::ArgMatches; -use futures::prelude::*; -use node_test_rig::{ - environment::{EnvironmentBuilder, LoggerConfig}, - testing_client_config, ClientGenesis, ValidatorFiles, -}; -use node_test_rig::{testing_validator_config, ClientConfig}; -use std::cmp::max; -use std::net::Ipv4Addr; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use types::{Epoch, EthSpec}; - -pub fn run_syncing_sim(matches: &ArgMatches) -> Result<(), String> { - let initial_delay = value_t!(matches, "initial_delay", u64).unwrap(); - let sync_timeout = value_t!(matches, "sync_timeout", u64).unwrap(); - let speed_up_factor = value_t!(matches, "speedup", u64).unwrap(); - let strategy = value_t!(matches, "strategy", String).unwrap(); - - println!("Syncing Simulator:"); - println!(" initial_delay:{}", initial_delay); - println!(" sync timeout: {}", sync_timeout); - println!(" speed up factor:{}", speed_up_factor); - println!(" strategy:{}", strategy); - - let log_level = "debug"; - let log_format = None; - - syncing_sim( - speed_up_factor, - initial_delay, - sync_timeout, - strategy, - log_level, - log_format, - ) -} - -fn syncing_sim( - speed_up_factor: u64, - initial_delay: u64, - sync_timeout: u64, - strategy: String, - log_level: &str, - log_format: Option<&str>, -) -> Result<(), String> { - let mut env = EnvironmentBuilder::minimal() - .initialize_logger(LoggerConfig { - path: None, - debug_level: String::from(log_level), - logfile_debug_level: String::from("debug"), - log_format: log_format.map(String::from), - logfile_format: None, - log_color: false, - disable_log_timestamp: false, - max_log_size: 0, - max_log_number: 0, - compression: false, - is_restricted: true, - sse_logging: false, - })? - .multi_threaded_tokio_runtime()? - .build()?; - - let spec = &mut env.eth2_config.spec; - let end_after_checks = true; - let eth1_block_time = Duration::from_millis(15_000 / speed_up_factor); - - // Set fork epochs to test syncing across fork boundaries - spec.altair_fork_epoch = Some(Epoch::new(1)); - spec.bellatrix_fork_epoch = Some(Epoch::new(2)); - spec.seconds_per_slot /= speed_up_factor; - spec.seconds_per_slot = max(1, spec.seconds_per_slot); - spec.eth1_follow_distance = 16; - spec.genesis_delay = eth1_block_time.as_secs() * spec.eth1_follow_distance * 2; - spec.min_genesis_time = 0; - spec.min_genesis_active_validator_count = 64; - spec.seconds_per_eth1_block = 1; - - let num_validators = 8; - let slot_duration = Duration::from_secs(spec.seconds_per_slot); - let context = env.core_context(); - let mut beacon_config = testing_client_config(); - - let genesis_time = SystemTime::now() - .duration_since(UNIX_EPOCH) - .map_err(|_| "should get system time")? - + Duration::from_secs(5); - beacon_config.genesis = ClientGenesis::Interop { - validator_count: num_validators, - genesis_time: genesis_time.as_secs(), - }; - beacon_config.dummy_eth1_backend = true; - beacon_config.sync_eth1_chain = true; - - beacon_config.network.enr_address = (Some(Ipv4Addr::LOCALHOST), None); - - // Generate the directories and keystores required for the validator clients. - let validator_indices = (0..num_validators).collect::>(); - let validator_files = ValidatorFiles::with_keystores(&validator_indices).unwrap(); - - let main_future = async { - /* - * Create a new `LocalNetwork` with one beacon node. - */ - let network = LocalNetwork::new(context, beacon_config.clone()).await?; - - /* - * Add a validator client which handles all validators from the genesis state. - */ - network - .add_validator_client(testing_validator_config(), 0, validator_files, true) - .await?; - - // Check all syncing strategies one after other. - pick_strategy( - &strategy, - network.clone(), - beacon_config.clone(), - slot_duration, - initial_delay, - sync_timeout, - ) - .await?; - - // The `final_future` either completes immediately or never completes, depending on the value - // of `end_after_checks`. - - if !end_after_checks { - future::pending::<()>().await; - } - - /* - * End the simulation by dropping the network. This will kill all running beacon nodes and - * validator clients. - */ - println!( - "Simulation complete. Finished with {} beacon nodes and {} validator clients", - network.beacon_node_count(), - network.validator_client_count() - ); - - // Be explicit about dropping the network, as this kills all the nodes. This ensures - // all the checks have adequate time to pass. - drop(network); - Ok::<(), String>(()) - }; - - env.runtime().block_on(main_future).unwrap(); - - env.fire_signal(); - env.shutdown_on_idle(); - - Ok(()) -} - -pub async fn pick_strategy( - strategy: &str, - network: LocalNetwork, - beacon_config: ClientConfig, - slot_duration: Duration, - initial_delay: u64, - sync_timeout: u64, -) -> Result<(), String> { - match strategy { - "one-node" => { - verify_one_node_sync( - network, - beacon_config, - slot_duration, - initial_delay, - sync_timeout, - ) - .await - } - "two-nodes" => { - verify_two_nodes_sync( - network, - beacon_config, - slot_duration, - initial_delay, - sync_timeout, - ) - .await - } - "mixed" => { - verify_in_between_sync( - network, - beacon_config, - slot_duration, - initial_delay, - sync_timeout, - ) - .await - } - "all" => { - verify_syncing( - network, - beacon_config, - slot_duration, - initial_delay, - sync_timeout, - ) - .await - } - _ => Err("Invalid strategy".into()), - } -} - -/// Verify one node added after `initial_delay` epochs is in sync -/// after `sync_timeout` epochs. -pub async fn verify_one_node_sync( - network: LocalNetwork, - beacon_config: ClientConfig, - slot_duration: Duration, - initial_delay: u64, - sync_timeout: u64, -) -> Result<(), String> { - let epoch_duration = slot_duration * (E::slots_per_epoch() as u32); - let network_c = network.clone(); - // Delay for `initial_delay` epochs before adding another node to start syncing - epoch_delay( - Epoch::new(initial_delay), - slot_duration, - E::slots_per_epoch(), - ) - .await; - // Add a beacon node - network.add_beacon_node(beacon_config, false).await?; - // Check every `epoch_duration` if nodes are synced - // limited to at most `sync_timeout` epochs - let mut interval = tokio::time::interval(epoch_duration); - let mut count = 0; - loop { - interval.tick().await; - if count >= sync_timeout || !check_still_syncing(&network_c).await? { - break; - } - count += 1; - } - let epoch = network.bootnode_epoch().await?; - verify_all_finalized_at(network, epoch) - .map_err(|e| format!("One node sync error: {}", e)) - .await -} - -/// Verify two nodes added after `initial_delay` epochs are in sync -/// after `sync_timeout` epochs. -pub async fn verify_two_nodes_sync( - network: LocalNetwork, - beacon_config: ClientConfig, - slot_duration: Duration, - initial_delay: u64, - sync_timeout: u64, -) -> Result<(), String> { - let epoch_duration = slot_duration * (E::slots_per_epoch() as u32); - let network_c = network.clone(); - // Delay for `initial_delay` epochs before adding another node to start syncing - epoch_delay( - Epoch::new(initial_delay), - slot_duration, - E::slots_per_epoch(), - ) - .await; - // Add beacon nodes - network - .add_beacon_node(beacon_config.clone(), false) - .await?; - network.add_beacon_node(beacon_config, false).await?; - // Check every `epoch_duration` if nodes are synced - // limited to at most `sync_timeout` epochs - let mut interval = tokio::time::interval(epoch_duration); - let mut count = 0; - loop { - interval.tick().await; - if count >= sync_timeout || !check_still_syncing(&network_c).await? { - break; - } - count += 1; - } - let epoch = network.bootnode_epoch().await?; - verify_all_finalized_at(network, epoch) - .map_err(|e| format!("One node sync error: {}", e)) - .await -} - -/// Add 2 syncing nodes after `initial_delay` epochs, -/// Add another node after `sync_timeout - 5` epochs and verify all are -/// in sync after `sync_timeout + 5` epochs. -pub async fn verify_in_between_sync( - network: LocalNetwork, - beacon_config: ClientConfig, - slot_duration: Duration, - initial_delay: u64, - sync_timeout: u64, -) -> Result<(), String> { - let epoch_duration = slot_duration * (E::slots_per_epoch() as u32); - let network_c = network.clone(); - // Delay for `initial_delay` epochs before adding another node to start syncing - let config1 = beacon_config.clone(); - epoch_delay( - Epoch::new(initial_delay), - slot_duration, - E::slots_per_epoch(), - ) - .await; - // Add two beacon nodes - network - .add_beacon_node(beacon_config.clone(), false) - .await?; - network.add_beacon_node(beacon_config, false).await?; - // Delay before adding additional syncing nodes. - epoch_delay( - Epoch::new(sync_timeout - 5), - slot_duration, - E::slots_per_epoch(), - ) - .await; - // Add a beacon node - network.add_beacon_node(config1.clone(), false).await?; - // Check every `epoch_duration` if nodes are synced - // limited to at most `sync_timeout` epochs - let mut interval = tokio::time::interval(epoch_duration); - let mut count = 0; - loop { - interval.tick().await; - if count >= sync_timeout || !check_still_syncing(&network_c).await? { - break; - } - count += 1; - } - let epoch = network.bootnode_epoch().await?; - verify_all_finalized_at(network, epoch) - .map_err(|e| format!("One node sync error: {}", e)) - .await -} - -/// Run syncing strategies one after other. -pub async fn verify_syncing( - network: LocalNetwork, - beacon_config: ClientConfig, - slot_duration: Duration, - initial_delay: u64, - sync_timeout: u64, -) -> Result<(), String> { - verify_one_node_sync( - network.clone(), - beacon_config.clone(), - slot_duration, - initial_delay, - sync_timeout, - ) - .await?; - println!("Completed one node sync"); - verify_two_nodes_sync( - network.clone(), - beacon_config.clone(), - slot_duration, - initial_delay, - sync_timeout, - ) - .await?; - println!("Completed two node sync"); - verify_in_between_sync( - network, - beacon_config, - slot_duration, - initial_delay, - sync_timeout, - ) - .await?; - println!("Completed in between sync"); - Ok(()) -} - -pub async fn check_still_syncing(network: &LocalNetwork) -> Result { - // get syncing status of nodes - let mut status = Vec::new(); - for remote_node in network.remote_nodes()? { - status.push( - remote_node - .get_node_syncing() - .await - .map(|body| body.data.is_syncing) - .map_err(|e| format!("Get syncing status via http failed: {:?}", e))?, - ) - } - Ok(status.iter().any(|is_syncing| *is_syncing)) -} diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index 445d4f1a5d9..06d484a52bb 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -892,6 +892,7 @@ impl UnsignedBlock { } } +#[derive(Debug)] pub enum SignedBlock { Full(PublishBlockRequest), Blinded(Arc>), From f7aca97a55fe33f51987b1b72851ff506c72f31a Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Tue, 23 Apr 2024 01:06:39 +0900 Subject: [PATCH 16/37] Handle sync lookup request streams in network context (#5583) * by-root-stream-terminator * Fix tests * Resolve merge conflicts * Log report reason * Some lints and bugfixes (#23) * fix lints * bug fixes * Fix tests * Merge branch 'unstable' of https://github.com/sigp/lighthouse into handle-sync-lookup-requests * Pr 5583 review (#24) * add bad state warn log * add rust docs to new fields in `SyncNetworkContext` * remove timestamp todo * add back lookup verify error * remove TODOs --- .../network/src/sync/block_lookups/common.rs | 156 ++------- .../network/src/sync/block_lookups/mod.rs | 317 ++++++++---------- .../src/sync/block_lookups/parent_lookup.rs | 70 +--- .../sync/block_lookups/single_block_lookup.rs | 72 ++-- .../network/src/sync/block_lookups/tests.rs | 13 +- beacon_node/network/src/sync/manager.rs | 203 ++++++----- .../network/src/sync/network_context.rs | 202 +++++++++-- .../src/sync/network_context/requests.rs | 149 ++++++++ 8 files changed, 643 insertions(+), 539 deletions(-) create mode 100644 beacon_node/network/src/sync/network_context/requests.rs diff --git a/beacon_node/network/src/sync/block_lookups/common.rs b/beacon_node/network/src/sync/block_lookups/common.rs index 8f7881eea8a..3bd39301b21 100644 --- a/beacon_node/network/src/sync/block_lookups/common.rs +++ b/beacon_node/network/src/sync/block_lookups/common.rs @@ -1,22 +1,21 @@ use crate::sync::block_lookups::parent_lookup::PARENT_FAIL_TOLERANCE; use crate::sync::block_lookups::single_block_lookup::{ - LookupRequestError, LookupVerifyError, SingleBlockLookup, SingleLookupRequestState, State, + LookupRequestError, SingleBlockLookup, SingleLookupRequestState, }; use crate::sync::block_lookups::{ BlobRequestState, BlockLookups, BlockRequestState, PeerId, SINGLE_BLOCK_LOOKUP_MAX_ATTEMPTS, }; use crate::sync::manager::{BlockProcessType, Id, SingleLookupReqId}; -use crate::sync::network_context::SyncNetworkContext; +use crate::sync::network_context::{ + BlobsByRootSingleBlockRequest, BlocksByRootSingleRequest, SyncNetworkContext, +}; use beacon_chain::block_verification_types::RpcBlock; use beacon_chain::data_availability_checker::ChildComponents; -use beacon_chain::{get_block_root, BeaconChainTypes}; -use lighthouse_network::rpc::methods::BlobsByRootRequest; -use lighthouse_network::rpc::BlocksByRootRequest; -use std::ops::IndexMut; +use beacon_chain::BeaconChainTypes; use std::sync::Arc; use std::time::Duration; -use types::blob_sidecar::{BlobIdentifier, FixedBlobSidecarList}; -use types::{BlobSidecar, ChainSpec, Hash256, SignedBeaconBlock}; +use types::blob_sidecar::FixedBlobSidecarList; +use types::{Hash256, SignedBeaconBlock}; #[derive(Debug, Copy, Clone)] pub enum ResponseType { @@ -73,9 +72,6 @@ pub trait RequestState { /// The type of the request . type RequestType; - /// A block or blob response. - type ResponseType; - /// The type created after validation. type VerifiedResponseType: Clone; @@ -85,14 +81,11 @@ pub trait RequestState { /* Request building methods */ /// Construct a new request. - fn build_request( - &mut self, - spec: &ChainSpec, - ) -> Result<(PeerId, Self::RequestType), LookupRequestError> { + fn build_request(&mut self) -> Result<(PeerId, Self::RequestType), LookupRequestError> { // Verify and construct request. self.too_many_attempts()?; let peer = self.get_peer()?; - let request = self.new_request(spec); + let request = self.new_request(); Ok((peer, request)) } @@ -100,7 +93,7 @@ pub trait RequestState { fn build_request_and_send( &mut self, id: Id, - cx: &SyncNetworkContext, + cx: &mut SyncNetworkContext, ) -> Result<(), LookupRequestError> { // Check if request is necessary. if !self.get_state().is_awaiting_download() { @@ -108,7 +101,7 @@ pub trait RequestState { } // Construct request. - let (peer_id, request) = self.build_request(&cx.chain.spec)?; + let (peer_id, request) = self.build_request()?; // Update request state. let req_counter = self.get_state_mut().on_download_start(peer_id); @@ -144,61 +137,18 @@ pub trait RequestState { } /// Initialize `Self::RequestType`. - fn new_request(&self, spec: &ChainSpec) -> Self::RequestType; + fn new_request(&self) -> Self::RequestType; /// Send the request to the network service. fn make_request( id: SingleLookupReqId, peer_id: PeerId, request: Self::RequestType, - cx: &SyncNetworkContext, + cx: &mut SyncNetworkContext, ) -> Result<(), LookupRequestError>; /* Response handling methods */ - /// Verify the response is valid based on what we requested. - fn verify_response( - &mut self, - expected_block_root: Hash256, - peer_id: PeerId, - response: Option, - ) -> Result, LookupVerifyError> { - let result = match *self.get_state().get_state() { - State::AwaitingDownload => Err(LookupVerifyError::ExtraBlocksReturned), - State::Downloading { peer_id: _ } => { - // TODO: We requested a download from Downloading { peer_id }, but the network - // injects a response from a different peer_id. What should we do? The peer_id to - // track for scoring is the one that actually sent the response, not the state's - self.verify_response_inner(expected_block_root, response) - } - State::Processing { .. } | State::Processed { .. } => match response { - // We sent the block for processing and received an extra block. - Some(_) => Err(LookupVerifyError::ExtraBlocksReturned), - // This is simply the stream termination and we are already processing the block - None => Ok(None), - }, - }; - - match result { - Ok(Some(response)) => { - self.get_state_mut().on_download_success(peer_id); - Ok(Some(response)) - } - Ok(None) => Ok(None), - Err(e) => { - self.get_state_mut().on_download_failure(); - Err(e) - } - } - } - - /// The response verification unique to block or blobs. - fn verify_response_inner( - &mut self, - expected_block_root: Hash256, - response: Option, - ) -> Result, LookupVerifyError>; - /// A getter for the parent root of the response. Returns an `Option` because we won't know /// the blob parent if we don't end up getting any blobs in the response. fn get_parent_root(verified_response: &Self::VerifiedResponseType) -> Option; @@ -247,49 +197,24 @@ pub trait RequestState { } impl RequestState for BlockRequestState { - type RequestType = BlocksByRootRequest; - type ResponseType = Arc>; + type RequestType = BlocksByRootSingleRequest; type VerifiedResponseType = Arc>; type ReconstructedResponseType = RpcBlock; - fn new_request(&self, spec: &ChainSpec) -> BlocksByRootRequest { - BlocksByRootRequest::new(vec![self.requested_block_root], spec) + fn new_request(&self) -> Self::RequestType { + BlocksByRootSingleRequest(self.requested_block_root) } fn make_request( id: SingleLookupReqId, peer_id: PeerId, request: Self::RequestType, - cx: &SyncNetworkContext, + cx: &mut SyncNetworkContext, ) -> Result<(), LookupRequestError> { cx.block_lookup_request(id, peer_id, request) .map_err(LookupRequestError::SendFailed) } - fn verify_response_inner( - &mut self, - expected_block_root: Hash256, - response: Option, - ) -> Result>>, LookupVerifyError> { - match response { - Some(block) => { - // Compute the block root using this specific function so that we can get timing - // metrics. - let block_root = get_block_root(&block); - if block_root != expected_block_root { - // return an error and drop the block - // NOTE: we take this is as a download failure to prevent counting the - // attempt as a chain failure, but simply a peer failure. - Err(LookupVerifyError::RootMismatch) - } else { - // Return the block for processing. - Ok(Some(block)) - } - } - None => Err(LookupVerifyError::NoBlockReturned), - } - } - fn get_parent_root(verified_response: &Arc>) -> Option { Some(verified_response.parent_root()) } @@ -340,60 +265,27 @@ impl RequestState for BlockRequestState } impl RequestState for BlobRequestState { - type RequestType = BlobsByRootRequest; - type ResponseType = Arc>; + type RequestType = BlobsByRootSingleBlockRequest; type VerifiedResponseType = FixedBlobSidecarList; type ReconstructedResponseType = FixedBlobSidecarList; - fn new_request(&self, spec: &ChainSpec) -> BlobsByRootRequest { - let blob_id_vec: Vec = self.requested_ids.clone().into(); - BlobsByRootRequest::new(blob_id_vec, spec) + fn new_request(&self) -> Self::RequestType { + BlobsByRootSingleBlockRequest { + block_root: self.block_root, + indices: self.requested_ids.indices(), + } } fn make_request( id: SingleLookupReqId, peer_id: PeerId, request: Self::RequestType, - cx: &SyncNetworkContext, + cx: &mut SyncNetworkContext, ) -> Result<(), LookupRequestError> { cx.blob_lookup_request(id, peer_id, request) .map_err(LookupRequestError::SendFailed) } - fn verify_response_inner( - &mut self, - expected_block_root: Hash256, - blob: Option, - ) -> Result>, LookupVerifyError> { - match blob { - Some(blob) => { - let received_id = blob.id(); - - if !self.requested_ids.contains(&received_id) { - return Err(LookupVerifyError::UnrequestedBlobId(received_id)); - } - if !blob.verify_blob_sidecar_inclusion_proof().unwrap_or(false) { - return Err(LookupVerifyError::InvalidInclusionProof); - } - if blob.block_root() != expected_block_root { - return Err(LookupVerifyError::UnrequestedHeader); - } - - // State should remain downloading until we receive the stream terminator. - self.requested_ids.remove(&received_id); - - // The inclusion proof check above ensures `blob.index` is < MAX_BLOBS_PER_BLOCK - let blob_index = blob.index; - *self.blob_download_queue.index_mut(blob_index as usize) = Some(blob); - Ok(None) - } - None => { - let blobs = std::mem::take(&mut self.blob_download_queue); - Ok(Some(blobs)) - } - } - } - fn get_parent_root(verified_response: &FixedBlobSidecarList) -> Option { verified_response .into_iter() diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index a5826bcb3d8..fa2683fb0f0 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -1,6 +1,6 @@ -use self::parent_lookup::ParentVerifyError; use self::single_block_lookup::SingleBlockLookup; use super::manager::BlockProcessingResult; +use super::network_context::{LookupFailure, LookupVerifyError}; use super::BatchProcessResult; use super::{manager::BlockProcessType, network_context::SyncNetworkContext}; use crate::metrics; @@ -21,7 +21,6 @@ pub use common::Lookup; pub use common::Parent; pub use common::RequestState; use fnv::FnvHashMap; -use lighthouse_network::rpc::RPCError; use lighthouse_network::{PeerAction, PeerId}; use lru_cache::LRUTimeCache; pub use single_block_lookup::{BlobRequestState, BlockRequestState}; @@ -133,7 +132,7 @@ impl BlockLookups { pub fn trigger_single_lookup( &mut self, mut single_block_lookup: SingleBlockLookup, - cx: &SyncNetworkContext, + cx: &mut SyncNetworkContext, ) { let block_root = single_block_lookup.block_root(); match single_block_lookup.request_block_and_blobs(cx) { @@ -319,40 +318,41 @@ impl BlockLookups { &mut self, lookup_id: SingleLookupReqId, peer_id: PeerId, - response: Option, + response: R::VerifiedResponseType, seen_timestamp: Duration, - cx: &SyncNetworkContext, + cx: &mut SyncNetworkContext, ) { let id = lookup_id.id; let response_type = R::response_type(); - let Some(lookup) = self.get_single_lookup::(lookup_id) else { - if response.is_some() { - // We don't have the ability to cancel in-flight RPC requests. So this can happen - // if we started this RPC request, and later saw the block/blobs via gossip. - debug!( - self.log, - "Block returned for single block lookup not present"; - "response_type" => ?response_type, - ); - } + let Some(mut lookup) = self.get_single_lookup::(lookup_id) else { + // We don't have the ability to cancel in-flight RPC requests. So this can happen + // if we started this RPC request, and later saw the block/blobs via gossip. + debug!( + self.log, + "Block returned for single block lookup not present"; + "response_type" => ?response_type, + ); return; }; let expected_block_root = lookup.block_root(); - if response.is_some() { - debug!(self.log, - "Peer returned response for single lookup"; - "peer_id" => %peer_id , - "id" => ?id, - "block_root" => ?expected_block_root, - "response_type" => ?response_type, - ); - } + debug!(self.log, + "Peer returned response for single lookup"; + "peer_id" => %peer_id , + "id" => ?id, + "block_root" => ?expected_block_root, + "response_type" => ?response_type, + ); - match self.single_lookup_response_inner::(peer_id, response, seen_timestamp, cx, lookup) - { - Ok(lookup) => { + match self.handle_verified_response::( + seen_timestamp, + cx, + BlockProcessType::SingleBlock { id: lookup.id }, + response, + &mut lookup, + ) { + Ok(_) => { self.single_block_lookups.insert(id, lookup); } Err(e) => { @@ -372,53 +372,10 @@ impl BlockLookups { /// Consolidates error handling for `single_lookup_response`. An `Err` here should always mean /// the lookup is dropped. - fn single_lookup_response_inner>( - &self, - peer_id: PeerId, - response: Option, - seen_timestamp: Duration, - cx: &SyncNetworkContext, - mut lookup: SingleBlockLookup, - ) -> Result, LookupRequestError> { - let response_type = R::response_type(); - let log = self.log.clone(); - let expected_block_root = lookup.block_root(); - let request_state = R::request_state_mut(&mut lookup); - - match request_state.verify_response(expected_block_root, peer_id, response) { - Ok(Some(verified_response)) => { - self.handle_verified_response::( - seen_timestamp, - cx, - BlockProcessType::SingleBlock { id: lookup.id }, - verified_response, - &mut lookup, - )?; - } - Ok(None) => {} - Err(e) => { - debug!( - log, - "Single lookup response verification failed, retrying"; - "block_root" => ?expected_block_root, - "peer_id" => %peer_id, - "response_type" => ?response_type, - "error" => ?e - ); - let msg = e.into(); - cx.report_peer(peer_id, PeerAction::LowToleranceError, msg); - - request_state.register_failure_downloading(); - lookup.request_block_and_blobs(cx)?; - } - } - Ok(lookup) - } - fn handle_verified_response>( &self, seen_timestamp: Duration, - cx: &SyncNetworkContext, + cx: &mut SyncNetworkContext, process_type: BlockProcessType, verified_response: R::VerifiedResponseType, lookup: &mut SingleBlockLookup, @@ -438,6 +395,10 @@ impl BlockLookups { }; if !delay_send { + R::request_state_mut(lookup) + .get_state_mut() + .on_download_success() + .map_err(LookupRequestError::BadState)?; self.send_block_for_processing( block_root, block, @@ -448,6 +409,10 @@ impl BlockLookups { } } CachedChild::DownloadIncomplete => { + R::request_state_mut(lookup) + .get_state_mut() + .on_download_success() + .map_err(LookupRequestError::BadState)?; // If this was the result of a block request, we can't determine if the block peer // did anything wrong. If we already had both a block and blobs response processed, // we should penalize the blobs peer because they did not provide all blobs on the @@ -458,14 +423,21 @@ impl BlockLookups { } lookup.request_block_and_blobs(cx)?; } - CachedChild::NotRequired => R::send_reconstructed_for_processing( - id, - self, - block_root, - R::verified_to_reconstructed(block_root, verified_response), - seen_timestamp, - cx, - )?, + CachedChild::NotRequired => { + R::request_state_mut(lookup) + .get_state_mut() + .on_download_success() + .map_err(LookupRequestError::BadState)?; + + R::send_reconstructed_for_processing( + id, + self, + block_root, + R::verified_to_reconstructed(block_root, verified_response), + seen_timestamp, + cx, + )? + } CachedChild::Err(e) => { warn!(self.log, "Consistency error in cached block"; "error" => ?e, @@ -511,26 +483,22 @@ impl BlockLookups { &mut self, id: SingleLookupReqId, peer_id: PeerId, - response: Option, + response: R::VerifiedResponseType, seen_timestamp: Duration, - cx: &SyncNetworkContext, + cx: &mut SyncNetworkContext, ) { let Some(mut parent_lookup) = self.get_parent_lookup::(id) else { - if response.is_some() { - debug!(self.log, "Response for a parent lookup request that was not found"; "peer_id" => %peer_id); - } + debug!(self.log, "Response for a parent lookup request that was not found"; "peer_id" => %peer_id); return; }; - if response.is_some() { - debug!(self.log, - "Peer returned response for parent lookup"; - "peer_id" => %peer_id , - "id" => ?id, - "block_root" => ?parent_lookup.current_parent_request.block_request_state.requested_block_root, - "response_type" => ?R::response_type(), - ); - } + debug!(self.log, + "Peer returned response for parent lookup"; + "peer_id" => %peer_id , + "id" => ?id, + "block_root" => ?parent_lookup.current_parent_request.block_request_state.requested_block_root, + "response_type" => ?R::response_type(), + ); match self.parent_lookup_response_inner::( peer_id, @@ -558,59 +526,17 @@ impl BlockLookups { fn parent_lookup_response_inner>( &mut self, peer_id: PeerId, - response: Option, + response: R::VerifiedResponseType, seen_timestamp: Duration, - cx: &SyncNetworkContext, - parent_lookup: &mut ParentLookup, - ) -> Result<(), RequestError> { - match parent_lookup.verify_response::(peer_id, response, &mut self.failed_chains) { - Ok(Some(verified_response)) => { - self.handle_verified_response::( - seen_timestamp, - cx, - BlockProcessType::ParentLookup { - chain_hash: parent_lookup.chain_hash(), - }, - verified_response, - &mut parent_lookup.current_parent_request, - )?; - } - Ok(None) => {} - Err(e) => self.handle_parent_verify_error::(peer_id, parent_lookup, e, cx)?, - }; - Ok(()) - } - - /// Handle logging and peer scoring for `ParentVerifyError`s during parent lookup requests. - fn handle_parent_verify_error>( - &mut self, - peer_id: PeerId, + cx: &mut SyncNetworkContext, parent_lookup: &mut ParentLookup, - e: ParentVerifyError, - cx: &SyncNetworkContext, ) -> Result<(), RequestError> { - match e { - ParentVerifyError::RootMismatch - | ParentVerifyError::NoBlockReturned - | ParentVerifyError::NotEnoughBlobsReturned - | ParentVerifyError::ExtraBlocksReturned - | ParentVerifyError::UnrequestedBlobId(_) - | ParentVerifyError::InvalidInclusionProof - | ParentVerifyError::UnrequestedHeader - | ParentVerifyError::ExtraBlobsReturned - | ParentVerifyError::InvalidIndex(_) => { - let e = e.into(); - warn!(self.log, "Peer sent invalid response to parent request"; - "peer_id" => %peer_id, "reason" => %e); - - // We do not tolerate these kinds of errors. We will accept a few but these are signs - // of a faulty peer. - cx.report_peer(peer_id, PeerAction::LowToleranceError, e); - - // We try again if possible. - parent_lookup.request_parent(cx)?; - } - ParentVerifyError::PreviousFailure { parent_root } => { + // check if the parent of this block isn't in the failed cache. If it is, this chain should + // be dropped and the peer downscored. + if let Some(parent_root) = R::get_parent_root(&response) { + if self.failed_chains.contains(&parent_root) { + let request_state = R::request_state_mut(&mut parent_lookup.current_parent_request); + request_state.register_failure_downloading(); debug!( self.log, "Parent chain ignored due to past failure"; @@ -624,8 +550,20 @@ impl BlockLookups { PeerAction::MidToleranceError, "bbroot_failed_chains", ); + return Ok(()); } } + + self.handle_verified_response::( + seen_timestamp, + cx, + BlockProcessType::ParentLookup { + chain_hash: parent_lookup.chain_hash(), + }, + response, + &mut parent_lookup.current_parent_request, + )?; + Ok(()) } @@ -665,8 +603,8 @@ impl BlockLookups { // This happens if the peer disconnects while the block is being // processed. Drop the request without extra penalty } - RequestError::BadState(_) => { - // Should never happen + RequestError::BadState(..) => { + warn!(self.log, "Failed to request parent"; "error" => e.as_static()); } } } @@ -675,12 +613,9 @@ impl BlockLookups { pub fn peer_disconnected(&mut self, peer_id: &PeerId, cx: &mut SyncNetworkContext) { /* Check disconnection for single lookups */ - self.single_block_lookups.retain(|id, req| { + self.single_block_lookups.retain(|_, req| { let should_drop_lookup = req.should_drop_lookup_on_disconnected_peer(peer_id, cx, &self.log); - if should_drop_lookup { - debug!(self.log, "Dropping lookup after peer disconnected"; "id" => id, "block_root" => %req.block_root()); - } !should_drop_lookup }); @@ -702,21 +637,28 @@ impl BlockLookups { &mut self, id: SingleLookupReqId, peer_id: &PeerId, - cx: &SyncNetworkContext, - error: RPCError, + cx: &mut SyncNetworkContext, + error: LookupFailure, ) { - let msg = error.as_static_str(); + // Only downscore lookup verify errors. RPC errors are downscored in the network handler. + if let LookupFailure::LookupVerifyError(e) = &error { + // Downscore peer even if lookup is not known + self.downscore_on_rpc_error(peer_id, e, cx); + } + let Some(mut parent_lookup) = self.get_parent_lookup::(id) else { debug!(self.log, "RPC failure for a block parent lookup request that was not found"; "peer_id" => %peer_id, - "error" => msg + "error" => %error ); return; }; R::request_state_mut(&mut parent_lookup.current_parent_request) .register_failure_downloading(); - trace!(self.log, "Parent lookup block request failed"; &parent_lookup, "error" => msg); + debug!(self.log, "Parent lookup block request failed"; + "chain_hash" => %parent_lookup.chain_hash(), "id" => ?id, "error" => %error + ); self.request_parent(parent_lookup, cx); @@ -731,13 +673,18 @@ impl BlockLookups { &mut self, id: SingleLookupReqId, peer_id: &PeerId, - cx: &SyncNetworkContext, - error: RPCError, + cx: &mut SyncNetworkContext, + error: LookupFailure, ) { - let msg = error.as_static_str(); + // Only downscore lookup verify errors. RPC errors are downscored in the network handler. + if let LookupFailure::LookupVerifyError(e) = &error { + // Downscore peer even if lookup is not known + self.downscore_on_rpc_error(peer_id, e, cx); + } + let log = self.log.clone(); let Some(mut lookup) = self.get_single_lookup::(id) else { - debug!(log, "Error response to dropped lookup"; "error" => ?error); + debug!(log, "Error response to dropped lookup"; "error" => %error); return; }; let block_root = lookup.block_root(); @@ -746,7 +693,7 @@ impl BlockLookups { trace!(log, "Single lookup failed"; "block_root" => ?block_root, - "error" => msg, + "error" => %error, "peer_id" => %peer_id, "response_type" => ?response_type ); @@ -758,7 +705,8 @@ impl BlockLookups { "error" => ?e, "block_root" => ?block_root, ); - self.single_block_lookups.remove(&id); + } else { + self.single_block_lookups.insert(id, lookup); } metrics::set_gauge( @@ -1006,20 +954,21 @@ impl BlockLookups { } BlockProcessingResult::Ok(AvailabilityProcessingStatus::Imported(_)) | BlockProcessingResult::Err(BlockError::BlockIsAlreadyKnown(_)) => { + let (chain_hash, blocks, hashes, block_request) = + parent_lookup.parts_for_processing(); + + let blocks = self.add_child_block_to_chain(chain_hash, blocks, cx).into(); + + let process_id = ChainSegmentProcessId::ParentLookup(chain_hash); + // Check if the beacon processor is available let Some(beacon_processor) = cx.beacon_processor_if_enabled() else { return trace!( self.log, "Dropping parent chain segment that was ready for processing."; - parent_lookup + "chain_hash" => %chain_hash, ); }; - let (chain_hash, blocks, hashes, block_request) = - parent_lookup.parts_for_processing(); - - let blocks = self.add_child_block_to_chain(chain_hash, blocks, cx).into(); - - let process_id = ChainSegmentProcessId::ParentLookup(chain_hash); match beacon_processor.send_chain_segment(process_id, blocks) { Ok(_) => { @@ -1073,7 +1022,7 @@ impl BlockLookups { &mut self, chain_hash: Hash256, mut blocks: VecDeque>, - cx: &SyncNetworkContext, + cx: &mut SyncNetworkContext, ) -> VecDeque> { // Find the child block that spawned the parent lookup request and add it to the chain // to send for processing. @@ -1126,12 +1075,16 @@ impl BlockLookups { fn handle_parent_block_error( &mut self, outcome: BlockError<::EthSpec>, - cx: &SyncNetworkContext, + cx: &mut SyncNetworkContext, mut parent_lookup: ParentLookup, ) { // We should always have a block peer. - let Ok(block_peer_id) = parent_lookup.block_processing_peer() else { - return; + let block_peer_id = match parent_lookup.block_processing_peer() { + Ok(peer_id) => peer_id, + Err(e) => { + warn!(self.log, "Parent lookup in bad state"; "chain_hash" => %parent_lookup.chain_hash(), "error" => e); + return; + } }; // We may not have a blob peer, if there were no blobs required for this block. @@ -1178,7 +1131,7 @@ impl BlockLookups { &mut self, chain_hash: Hash256, result: BatchProcessResult, - cx: &SyncNetworkContext, + cx: &mut SyncNetworkContext, ) { let Some((_hashes, request)) = self.processing_parent_lookups.remove(&chain_hash) else { return debug!(self.log, "Chain process response for a parent lookup request that was not found"; "chain_hash" => %chain_hash, "result" => ?result); @@ -1339,7 +1292,11 @@ impl BlockLookups { /// Attempts to request the next unknown parent. This method handles peer scoring and dropping /// the lookup in the event of failure. - fn request_parent(&mut self, mut parent_lookup: ParentLookup, cx: &SyncNetworkContext) { + fn request_parent( + &mut self, + mut parent_lookup: ParentLookup, + cx: &mut SyncNetworkContext, + ) { let response = parent_lookup.request_parent(cx); match response { @@ -1368,6 +1325,20 @@ impl BlockLookups { self.parent_lookups.drain(..).len() } + pub fn downscore_on_rpc_error( + &self, + peer_id: &PeerId, + error: &LookupVerifyError, + cx: &SyncNetworkContext, + ) { + // Note: logging the report event here with the full error display. The log inside + // `report_peer` only includes a smaller string, like "invalid_data" + let error_str: &'static str = error.into(); + + debug!(self.log, "reporting peer for sync lookup error"; "error" => error_str); + cx.report_peer(*peer_id, PeerAction::LowToleranceError, error_str); + } + pub fn update_metrics(&self) { metrics::set_gauge( &metrics::SYNC_SINGLE_BLOCK_LOOKUPS, diff --git a/beacon_node/network/src/sync/block_lookups/parent_lookup.rs b/beacon_node/network/src/sync/block_lookups/parent_lookup.rs index 55dd26b661e..b7a71860bff 100644 --- a/beacon_node/network/src/sync/block_lookups/parent_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/parent_lookup.rs @@ -1,7 +1,6 @@ -use super::single_block_lookup::{LookupRequestError, LookupVerifyError, SingleBlockLookup}; +use super::single_block_lookup::{LookupRequestError, SingleBlockLookup}; use super::{DownloadedBlock, PeerId}; use crate::sync::block_lookups::common::Parent; -use crate::sync::block_lookups::common::RequestState; use crate::sync::{manager::SLOT_IMPORT_TOLERANCE, network_context::SyncNetworkContext}; use beacon_chain::block_verification_types::AsBlock; use beacon_chain::block_verification_types::RpcBlock; @@ -10,8 +9,6 @@ use beacon_chain::BeaconChainTypes; use std::collections::VecDeque; use std::sync::Arc; use store::Hash256; -use strum::IntoStaticStr; -use types::blob_sidecar::BlobIdentifier; /// How many attempts we try to find a parent of a block before we give up trying. pub(crate) const PARENT_FAIL_TOLERANCE: u8 = 5; @@ -30,22 +27,8 @@ pub(crate) struct ParentLookup { pub current_parent_request: SingleBlockLookup, } -#[derive(Debug, PartialEq, Eq, IntoStaticStr)] -pub enum ParentVerifyError { - RootMismatch, - NoBlockReturned, - NotEnoughBlobsReturned, - ExtraBlocksReturned, - UnrequestedBlobId(BlobIdentifier), - InvalidInclusionProof, - UnrequestedHeader, - ExtraBlobsReturned, - InvalidIndex(u64), - PreviousFailure { parent_root: Hash256 }, -} - #[derive(Debug, PartialEq, Eq)] -pub enum RequestError { +pub(crate) enum RequestError { SendFailed(&'static str), ChainTooLong, /// We witnessed too many failures trying to complete this parent lookup. @@ -92,7 +75,7 @@ impl ParentLookup { } /// Attempts to request the next unknown parent. If the request fails, it should be removed. - pub fn request_parent(&mut self, cx: &SyncNetworkContext) -> Result<(), RequestError> { + pub fn request_parent(&mut self, cx: &mut SyncNetworkContext) -> Result<(), RequestError> { // check to make sure this request hasn't failed if self.downloaded_blocks.len() + 1 >= PARENT_DEPTH_TOLERANCE { return Err(RequestError::ChainTooLong); @@ -186,34 +169,6 @@ impl ParentLookup { } } - /// Verifies that the received block is what we requested. If so, parent lookup now waits for - /// the processing result of the block. - pub fn verify_response>( - &mut self, - peer_id: PeerId, - block: Option, - failed_chains: &mut lru_cache::LRUTimeCache, - ) -> Result, ParentVerifyError> { - let expected_block_root = self.current_parent_request.block_root(); - let request_state = R::request_state_mut(&mut self.current_parent_request); - let root_and_verified = - request_state.verify_response(expected_block_root, peer_id, block)?; - - // check if the parent of this block isn't in the failed cache. If it is, this chain should - // be dropped and the peer downscored. - if let Some(parent_root) = root_and_verified - .as_ref() - .and_then(|block| R::get_parent_root(block)) - { - if failed_chains.contains(&parent_root) { - request_state.register_failure_downloading(); - return Err(ParentVerifyError::PreviousFailure { parent_root }); - } - } - - Ok(root_and_verified) - } - pub fn add_peer(&mut self, peer: PeerId) { self.current_parent_request.add_peer(peer) } @@ -228,23 +183,6 @@ impl ParentLookup { } } -impl From for ParentVerifyError { - fn from(e: LookupVerifyError) -> Self { - use LookupVerifyError as E; - match e { - E::RootMismatch => ParentVerifyError::RootMismatch, - E::NoBlockReturned => ParentVerifyError::NoBlockReturned, - E::ExtraBlocksReturned => ParentVerifyError::ExtraBlocksReturned, - E::UnrequestedBlobId(blob_id) => ParentVerifyError::UnrequestedBlobId(blob_id), - E::InvalidInclusionProof => ParentVerifyError::InvalidInclusionProof, - E::UnrequestedHeader => ParentVerifyError::UnrequestedHeader, - E::ExtraBlobsReturned => ParentVerifyError::ExtraBlobsReturned, - E::InvalidIndex(index) => ParentVerifyError::InvalidIndex(index), - E::NotEnoughBlobsReturned => ParentVerifyError::NotEnoughBlobsReturned, - } - } -} - impl From for RequestError { fn from(e: LookupRequestError) -> Self { use LookupRequestError as E; @@ -282,7 +220,7 @@ impl RequestError { } RequestError::TooManyAttempts { cannot_process: _ } => "too_many_downloading_attempts", RequestError::NoPeers => "no_peers", - RequestError::BadState(_) => "bad_state", + RequestError::BadState(..) => "bad_state", } } } diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index 15d10c77c24..5bb663967d7 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -11,29 +11,16 @@ use beacon_chain::BeaconChainTypes; use itertools::Itertools; use lighthouse_network::PeerAction; use rand::seq::IteratorRandom; -use slog::{trace, Logger}; +use slog::{debug, Logger}; use std::collections::HashSet; use std::fmt::Debug; use std::marker::PhantomData; use std::sync::Arc; use store::Hash256; use strum::IntoStaticStr; -use types::blob_sidecar::{BlobIdentifier, FixedBlobSidecarList}; +use types::blob_sidecar::FixedBlobSidecarList; use types::EthSpec; -#[derive(Debug, PartialEq, Eq, IntoStaticStr)] -pub enum LookupVerifyError { - RootMismatch, - NoBlockReturned, - ExtraBlocksReturned, - UnrequestedBlobId(BlobIdentifier), - InvalidInclusionProof, - UnrequestedHeader, - ExtraBlobsReturned, - NotEnoughBlobsReturned, - InvalidIndex(u64), -} - #[derive(Debug, PartialEq, Eq, IntoStaticStr)] pub enum LookupRequestError { /// Too many failed attempts @@ -88,6 +75,7 @@ impl SingleBlockLookup { /// the next parent. pub fn update_requested_parent_block(&mut self, block_root: Hash256) { self.block_request_state.requested_block_root = block_root; + self.blob_request_state.block_root = block_root; self.block_request_state.state.state = State::AwaitingDownload; self.blob_request_state.state.state = State::AwaitingDownload; self.child_components = Some(ChildComponents::empty(block_root)); @@ -108,7 +96,7 @@ impl SingleBlockLookup { /// downloading the block and/or blobs. pub fn request_block_and_blobs( &mut self, - cx: &SyncNetworkContext, + cx: &mut SyncNetworkContext, ) -> Result<(), LookupRequestError> { let block_already_downloaded = self.block_already_downloaded(); let blobs_already_downloaded = self.blobs_already_downloaded(); @@ -216,7 +204,7 @@ impl SingleBlockLookup { pub fn should_drop_lookup_on_disconnected_peer( &mut self, peer_id: &PeerId, - cx: &SyncNetworkContext, + cx: &mut SyncNetworkContext, log: &Logger, ) -> bool { let block_root = self.block_root(); @@ -233,7 +221,7 @@ impl SingleBlockLookup { if block_peer_disconnected || blob_peer_disconnected { if let Err(e) = self.request_block_and_blobs(cx) { - trace!(log, "Single lookup failed on peer disconnection"; "block_root" => ?block_root, "error" => ?e); + debug!(log, "Single lookup failed on peer disconnection"; "block_root" => ?block_root, "error" => ?e); return true; } } @@ -318,6 +306,7 @@ pub struct BlobRequestState { /// from both block/blobs downloaded in the network layer and any blocks/blobs that exist in /// the data availability checker. pub requested_ids: MissingBlobs, + pub block_root: Hash256, /// Where we store blobs until we receive the stream terminator. pub blob_download_queue: FixedBlobSidecarList, pub state: SingleLookupRequestState, @@ -328,6 +317,7 @@ impl BlobRequestState { pub fn new(block_root: Hash256, peer_source: &[PeerId], is_deneb: bool) -> Self { let default_ids = MissingBlobs::new_without_block(block_root, is_deneb); Self { + block_root, requested_ids: default_ids, blob_download_queue: <_>::default(), state: SingleLookupRequestState::new(peer_source), @@ -416,11 +406,6 @@ impl SingleLookupRequestState { } } - // TODO: Should not leak the enum state - pub fn get_state(&self) -> &State { - &self.state - } - pub fn is_current_req_counter(&self, req_counter: u32) -> bool { self.req_counter == req_counter } @@ -460,8 +445,16 @@ impl SingleLookupRequestState { self.state = State::AwaitingDownload; } - pub fn on_download_success(&mut self, peer_id: PeerId) { - self.state = State::Processing { peer_id }; + pub fn on_download_success(&mut self) -> Result<(), String> { + match &self.state { + State::Downloading { peer_id } => { + self.state = State::Processing { peer_id: *peer_id }; + Ok(()) + } + other => Err(format!( + "request bad state, expected downloading got {other}" + )), + } } /// Registers a failure in processing a block. @@ -669,19 +662,9 @@ mod tests { ); as RequestState>::build_request( &mut sl.block_request_state, - &spec, ) .unwrap(); sl.block_request_state.state.state = State::Downloading { peer_id }; - - as RequestState>::verify_response( - &mut sl.block_request_state, - block.canonical_root(), - peer_id, - Some(block.into()), - ) - .unwrap() - .unwrap(); } #[test] @@ -714,7 +697,6 @@ mod tests { for _ in 1..TestLookup2::MAX_ATTEMPTS { as RequestState>::build_request( &mut sl.block_request_state, - &spec, ) .unwrap(); sl.block_request_state.state.on_download_failure(); @@ -723,30 +705,20 @@ mod tests { // Now we receive the block and send it for processing as RequestState>::build_request( &mut sl.block_request_state, - &spec, ) .unwrap(); sl.block_request_state.state.state = State::Downloading { peer_id }; - as RequestState>::verify_response( - &mut sl.block_request_state, - block.canonical_root(), - peer_id, - Some(block.into()), - ) - .unwrap() - .unwrap(); - // One processing failure maxes the available attempts sl.block_request_state.state.on_processing_failure(); assert_eq!( as RequestState>::build_request( &mut sl.block_request_state, - &spec - ), - Err(LookupRequestError::TooManyAttempts { + ) + .unwrap_err(), + LookupRequestError::TooManyAttempts { cannot_process: false - }) + } ) } } diff --git a/beacon_node/network/src/sync/block_lookups/tests.rs b/beacon_node/network/src/sync/block_lookups/tests.rs index 6d50fe63200..8e3b35ee5d3 100644 --- a/beacon_node/network/src/sync/block_lookups/tests.rs +++ b/beacon_node/network/src/sync/block_lookups/tests.rs @@ -15,9 +15,10 @@ use beacon_chain::test_utils::{ build_log, generate_rand_block_and_blobs, BeaconChainHarness, EphemeralHarnessType, NumBlobs, }; use beacon_processor::WorkEvent; -use lighthouse_network::rpc::RPCResponseErrorCode; +use lighthouse_network::rpc::{RPCError, RPCResponseErrorCode}; use lighthouse_network::types::SyncState; use lighthouse_network::{NetworkGlobals, Request}; +use slog::info; use slot_clock::{ManualSlotClock, SlotClock, TestingSlotClock}; use store::MemoryStore; use tokio::sync::mpsc; @@ -67,6 +68,7 @@ struct TestRig { /// `rng` for generating test blocks and blobs. rng: XorShiftRng, fork_name: ForkName, + log: Logger, } const D: Duration = Duration::new(0, 0); @@ -124,6 +126,7 @@ impl TestRig { log.clone(), ), fork_name, + log, } } @@ -136,6 +139,10 @@ impl TestRig { } } + fn log(&self, msg: &str) { + info!(self.log, "TEST_RIG"; "msg" => msg); + } + fn after_deneb(&self) -> bool { matches!(self.fork_name, ForkName::Deneb | ForkName::Electra) } @@ -914,7 +921,7 @@ fn test_parent_lookup_too_many_processing_attempts_must_blacklist() { // Trigger the request rig.trigger_unknown_parent_block(peer_id, block.into()); - // Fail downloading the block + rig.log("Fail downloading the block"); for i in 0..(parent_lookup::PARENT_FAIL_TOLERANCE - PROCESSING_FAILURES) { let id = rig.expect_block_parent_request(parent_root); // Blobs are only requested in the first iteration as this test only retries blocks @@ -925,7 +932,7 @@ fn test_parent_lookup_too_many_processing_attempts_must_blacklist() { rig.parent_lookup_failed_unavailable(id, peer_id); } - // Now fail processing a block in the parent request + rig.log("Now fail processing a block in the parent request"); for i in 0..PROCESSING_FAILURES { let id = rig.expect_block_parent_request(parent_root); // Blobs are only requested in the first iteration as this test only retries blocks diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index a868a092d3d..23bd1010bfe 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -36,7 +36,7 @@ use super::backfill_sync::{BackFillSync, ProcessResult, SyncStart}; use super::block_lookups::common::LookupType; use super::block_lookups::BlockLookups; -use super::network_context::{BlockOrBlob, RangeRequestId, SyncNetworkContext}; +use super::network_context::{BlockOrBlob, RangeRequestId, RpcEvent, SyncNetworkContext}; use super::peer_sync_info::{remote_sync_type, PeerSyncType}; use super::range_sync::{RangeSync, RangeSyncType, EPOCHS_PER_BATCH}; use crate::network_beacon_processor::{ChainSegmentProcessId, NetworkBeaconProcessor}; @@ -320,42 +320,12 @@ impl SyncManager { fn inject_error(&mut self, peer_id: PeerId, request_id: RequestId, error: RPCError) { trace!(self.log, "Sync manager received a failed RPC"); match request_id { - RequestId::SingleBlock { id } => match id.lookup_type { - LookupType::Current => self - .block_lookups - .single_block_lookup_failed::>( - id, - &peer_id, - &self.network, - error, - ), - LookupType::Parent => self - .block_lookups - .parent_lookup_failed::>( - id, - &peer_id, - &self.network, - error, - ), - }, - RequestId::SingleBlob { id } => match id.lookup_type { - LookupType::Current => self - .block_lookups - .single_block_lookup_failed::>( - id, - &peer_id, - &self.network, - error, - ), - LookupType::Parent => self - .block_lookups - .parent_lookup_failed::>( - id, - &peer_id, - &self.network, - error, - ), - }, + RequestId::SingleBlock { id } => { + self.on_single_block_response(id, peer_id, RpcEvent::RPCError(error)) + } + RequestId::SingleBlob { id } => { + self.on_single_blob_response(id, peer_id, RpcEvent::RPCError(error)) + } RequestId::RangeBlockAndBlobs { id } => { if let Some(sender_id) = self.network.range_request_failed(id) { match sender_id { @@ -694,7 +664,7 @@ impl SyncManager { } ChainSegmentProcessId::ParentLookup(chain_hash) => self .block_lookups - .parent_chain_processed(chain_hash, result, &self.network), + .parent_chain_processed(chain_hash, result, &mut self.network), }, } } @@ -836,26 +806,14 @@ impl SyncManager { seen_timestamp: Duration, ) { match request_id { - RequestId::SingleBlock { id } => match id.lookup_type { - LookupType::Current => self - .block_lookups - .single_lookup_response::>( - id, - peer_id, - block, - seen_timestamp, - &self.network, - ), - LookupType::Parent => self - .block_lookups - .parent_lookup_response::>( - id, - peer_id, - block, - seen_timestamp, - &self.network, - ), - }, + RequestId::SingleBlock { id } => self.on_single_block_response( + id, + peer_id, + match block { + Some(block) => RpcEvent::Response(block, seen_timestamp), + None => RpcEvent::StreamTermination, + }, + ), RequestId::SingleBlob { .. } => { crit!(self.log, "Block received during blob request"; "peer_id" => %peer_id ); } @@ -865,6 +823,56 @@ impl SyncManager { } } + fn on_single_block_response( + &mut self, + id: SingleLookupReqId, + peer_id: PeerId, + block: RpcEvent>>, + ) { + if let Some(resp) = self.network.on_single_block_response(id, block) { + match resp { + Ok((block, seen_timestamp)) => match id.lookup_type { + LookupType::Current => self + .block_lookups + .single_lookup_response::>( + id, + peer_id, + block, + seen_timestamp, + &mut self.network, + ), + LookupType::Parent => self + .block_lookups + .parent_lookup_response::>( + id, + peer_id, + block, + seen_timestamp, + &mut self.network, + ), + }, + Err(error) => match id.lookup_type { + LookupType::Current => self + .block_lookups + .single_block_lookup_failed::>( + id, + &peer_id, + &mut self.network, + error, + ), + LookupType::Parent => self + .block_lookups + .parent_lookup_failed::>( + id, + &peer_id, + &mut self.network, + error, + ), + }, + } + } + } + fn rpc_blob_received( &mut self, request_id: RequestId, @@ -876,32 +884,71 @@ impl SyncManager { RequestId::SingleBlock { .. } => { crit!(self.log, "Single blob received during block request"; "peer_id" => %peer_id ); } - RequestId::SingleBlob { id } => match id.lookup_type { - LookupType::Current => self - .block_lookups - .single_lookup_response::>( - id, - peer_id, - blob, - seen_timestamp, - &self.network, - ), - LookupType::Parent => self - .block_lookups - .parent_lookup_response::>( - id, - peer_id, - blob, - seen_timestamp, - &self.network, - ), - }, + RequestId::SingleBlob { id } => self.on_single_blob_response( + id, + peer_id, + match blob { + Some(blob) => RpcEvent::Response(blob, seen_timestamp), + None => RpcEvent::StreamTermination, + }, + ), RequestId::RangeBlockAndBlobs { id } => { self.range_block_and_blobs_response(id, peer_id, blob.into()) } } } + fn on_single_blob_response( + &mut self, + id: SingleLookupReqId, + peer_id: PeerId, + blob: RpcEvent>>, + ) { + if let Some(resp) = self.network.on_single_blob_response(id, blob) { + match resp { + Ok((blobs, seen_timestamp)) => match id.lookup_type { + LookupType::Current => self + .block_lookups + .single_lookup_response::>( + id, + peer_id, + blobs, + seen_timestamp, + &mut self.network, + ), + LookupType::Parent => self + .block_lookups + .parent_lookup_response::>( + id, + peer_id, + blobs, + seen_timestamp, + &mut self.network, + ), + }, + + Err(error) => match id.lookup_type { + LookupType::Current => self + .block_lookups + .single_block_lookup_failed::>( + id, + &peer_id, + &mut self.network, + error, + ), + LookupType::Parent => self + .block_lookups + .parent_lookup_failed::>( + id, + &peer_id, + &mut self.network, + error, + ), + }, + } + } + } + /// Handles receiving a response for a range sync request that should have both blocks and /// blobs. fn range_block_and_blobs_response( diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 96f8de46fb7..fc91270c1dc 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -1,6 +1,8 @@ //! Provides network functionality for the Syncing thread. This fundamentally wraps a network //! channel and stores a global RPC ID to perform requests. +use self::requests::{ActiveBlobsByRootRequest, ActiveBlocksByRootRequest}; +pub use self::requests::{BlobsByRootSingleBlockRequest, BlocksByRootSingleRequest}; use super::block_sidecar_coupling::BlocksAndBlobsRequestInfo; use super::manager::{Id, RequestId as SyncRequestId}; use super::range_sync::{BatchId, ByRangeRequestType, ChainId}; @@ -9,17 +11,23 @@ use crate::service::{NetworkMessage, RequestId}; use crate::status::ToStatusMessage; use crate::sync::manager::SingleLookupReqId; use beacon_chain::block_verification_types::RpcBlock; +use beacon_chain::validator_monitor::timestamp_now; use beacon_chain::{BeaconChain, BeaconChainTypes, EngineState}; use fnv::FnvHashMap; -use lighthouse_network::rpc::methods::{BlobsByRangeRequest, BlobsByRootRequest}; -use lighthouse_network::rpc::{BlocksByRangeRequest, BlocksByRootRequest, GoodbyeReason}; +use lighthouse_network::rpc::methods::BlobsByRangeRequest; +use lighthouse_network::rpc::{BlocksByRangeRequest, GoodbyeReason, RPCError}; use lighthouse_network::{Client, NetworkGlobals, PeerAction, PeerId, ReportSource, Request}; +pub use requests::LookupVerifyError; use slog::{debug, trace, warn}; use std::collections::hash_map::Entry; use std::sync::Arc; +use std::time::Duration; use tokio::sync::mpsc; +use types::blob_sidecar::FixedBlobSidecarList; use types::{BlobSidecar, EthSpec, SignedBeaconBlock}; +mod requests; + pub struct BlocksAndBlobsByRangeResponse { pub sender_id: RangeRequestId, pub responses: Result>, String>, @@ -37,6 +45,41 @@ pub enum RangeRequestId { }, } +#[derive(Debug)] +pub enum RpcEvent { + StreamTermination, + Response(T, Duration), + RPCError(RPCError), +} + +pub type RpcProcessingResult = Option>; + +pub enum LookupFailure { + RpcError(RPCError), + LookupVerifyError(LookupVerifyError), +} + +impl std::fmt::Display for LookupFailure { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + match self { + LookupFailure::RpcError(e) => write!(f, "RPC Error: {:?}", e), + LookupFailure::LookupVerifyError(e) => write!(f, "Lookup Verify Error: {:?}", e), + } + } +} + +impl From for LookupFailure { + fn from(e: RPCError) -> Self { + LookupFailure::RpcError(e) + } +} + +impl From for LookupFailure { + fn from(e: LookupVerifyError) -> Self { + LookupFailure::LookupVerifyError(e) + } +} + /// Wraps a Network channel to employ various RPC related network functionality for the Sync manager. This includes management of a global RPC request Id. pub struct SyncNetworkContext { /// The network channel to relay messages to the Network service. @@ -45,6 +88,12 @@ pub struct SyncNetworkContext { /// A sequential ID for all RPC requests. request_id: Id, + /// A mapping of active BlocksByRoot requests, including both current slot and parent lookups. + blocks_by_root_requests: FnvHashMap, + + /// A mapping of active BlobsByRoot requests, including both current slot and parent lookups. + blobs_by_root_requests: FnvHashMap>, + /// BlocksByRange requests paired with BlobsByRange range_blocks_and_blobs_requests: FnvHashMap)>, @@ -91,6 +140,8 @@ impl SyncNetworkContext { network_send, execution_engine_state: EngineState::Online, // always assume `Online` at the start request_id: 1, + blocks_by_root_requests: <_>::default(), + blobs_by_root_requests: <_>::default(), range_blocks_and_blobs_requests: FnvHashMap::default(), network_beacon_processor, chain, @@ -245,62 +296,57 @@ impl SyncNetworkContext { } pub fn block_lookup_request( - &self, + &mut self, id: SingleLookupReqId, peer_id: PeerId, - request: BlocksByRootRequest, + request: BlocksByRootSingleRequest, ) -> Result<(), &'static str> { debug!( self.log, "Sending BlocksByRoot Request"; "method" => "BlocksByRoot", - "block_roots" => ?request.block_roots().to_vec(), + "block_root" => ?request.0, "peer" => %peer_id, "id" => ?id ); self.send_network_msg(NetworkMessage::SendRequest { peer_id, - request: Request::BlocksByRoot(request), + request: Request::BlocksByRoot(request.into_request(&self.chain.spec)), request_id: RequestId::Sync(SyncRequestId::SingleBlock { id }), })?; + + self.blocks_by_root_requests + .insert(id, ActiveBlocksByRootRequest::new(request)); + Ok(()) } pub fn blob_lookup_request( - &self, + &mut self, id: SingleLookupReqId, - blob_peer_id: PeerId, - blob_request: BlobsByRootRequest, + peer_id: PeerId, + request: BlobsByRootSingleBlockRequest, ) -> Result<(), &'static str> { - if let Some(block_root) = blob_request - .blob_ids - .as_slice() - .first() - .map(|id| id.block_root) - { - let indices = blob_request - .blob_ids - .as_slice() - .iter() - .map(|id| id.index) - .collect::>(); - debug!( - self.log, - "Sending BlobsByRoot Request"; - "method" => "BlobsByRoot", - "block_root" => ?block_root, - "blob_indices" => ?indices, - "peer" => %blob_peer_id, - "id" => ?id - ); + debug!( + self.log, + "Sending BlobsByRoot Request"; + "method" => "BlobsByRoot", + "block_root" => ?request.block_root, + "blob_indices" => ?request.indices, + "peer" => %peer_id, + "id" => ?id + ); + + self.send_network_msg(NetworkMessage::SendRequest { + peer_id, + request: Request::BlobsByRoot(request.clone().into_request(&self.chain.spec)), + request_id: RequestId::Sync(SyncRequestId::SingleBlob { id }), + })?; + + self.blobs_by_root_requests + .insert(id, ActiveBlobsByRootRequest::new(request)); - self.send_network_msg(NetworkMessage::SendRequest { - peer_id: blob_peer_id, - request: Request::BlobsByRoot(blob_request), - request_id: RequestId::Sync(SyncRequestId::SingleBlob { id }), - })?; - } Ok(()) } @@ -329,7 +375,7 @@ impl SyncNetworkContext { /// Reports to the scoring algorithm the behaviour of a peer. pub fn report_peer(&self, peer_id: PeerId, action: PeerAction, msg: &'static str) { - debug!(self.log, "Sync reporting peer"; "peer_id" => %peer_id, "action" => %action); + debug!(self.log, "Sync reporting peer"; "peer_id" => %peer_id, "action" => %action, "msg" => %msg); self.network_send .send(NetworkMessage::ReportPeer { peer_id, @@ -405,4 +451,86 @@ impl SyncNetworkContext { self.range_blocks_and_blobs_requests .insert(id, (sender_id, info)); } + + // Request handlers + + pub fn on_single_block_response( + &mut self, + request_id: SingleLookupReqId, + block: RpcEvent>>, + ) -> RpcProcessingResult>> { + let Entry::Occupied(mut request) = self.blocks_by_root_requests.entry(request_id) else { + return None; + }; + + Some(match block { + RpcEvent::Response(block, seen_timestamp) => { + match request.get_mut().add_response(block) { + Ok(block) => Ok((block, seen_timestamp)), + Err(e) => { + // The request must be dropped after receiving an error. + request.remove(); + Err(e.into()) + } + } + } + RpcEvent::StreamTermination => match request.remove().terminate() { + Ok(_) => return None, + Err(e) => Err(e.into()), + }, + RpcEvent::RPCError(e) => { + request.remove(); + Err(e.into()) + } + }) + } + + pub fn on_single_blob_response( + &mut self, + request_id: SingleLookupReqId, + blob: RpcEvent>>, + ) -> RpcProcessingResult> { + let Entry::Occupied(mut request) = self.blobs_by_root_requests.entry(request_id) else { + return None; + }; + + Some(match blob { + RpcEvent::Response(blob, _) => match request.get_mut().add_response(blob) { + Ok(Some(blobs)) => to_fixed_blob_sidecar_list(blobs) + .map(|blobs| (blobs, timestamp_now())) + .map_err(Into::into), + Ok(None) => return None, + Err(e) => { + request.remove(); + Err(e.into()) + } + }, + RpcEvent::StreamTermination => { + // Stream terminator + match request.remove().terminate() { + Some(blobs) => to_fixed_blob_sidecar_list(blobs) + .map(|blobs| (blobs, timestamp_now())) + .map_err(Into::into), + None => return None, + } + } + RpcEvent::RPCError(e) => { + request.remove(); + Err(e.into()) + } + }) + } +} + +fn to_fixed_blob_sidecar_list( + blobs: Vec>>, +) -> Result, LookupVerifyError> { + let mut fixed_list = FixedBlobSidecarList::default(); + for blob in blobs.into_iter() { + let index = blob.index as usize; + *fixed_list + .get_mut(index) + .ok_or(LookupVerifyError::UnrequestedBlobIndex(index as u64))? = Some(blob) + } + Ok(fixed_list) } diff --git a/beacon_node/network/src/sync/network_context/requests.rs b/beacon_node/network/src/sync/network_context/requests.rs new file mode 100644 index 00000000000..0522b7fa384 --- /dev/null +++ b/beacon_node/network/src/sync/network_context/requests.rs @@ -0,0 +1,149 @@ +use beacon_chain::get_block_root; +use lighthouse_network::rpc::{methods::BlobsByRootRequest, BlocksByRootRequest}; +use std::sync::Arc; +use strum::IntoStaticStr; +use types::{ + blob_sidecar::BlobIdentifier, BlobSidecar, ChainSpec, EthSpec, Hash256, SignedBeaconBlock, +}; + +#[derive(Debug, PartialEq, Eq, IntoStaticStr)] +pub enum LookupVerifyError { + NoResponseReturned, + TooManyResponses, + UnrequestedBlockRoot(Hash256), + UnrequestedBlobIndex(u64), + InvalidInclusionProof, + DuplicateData, +} + +pub struct ActiveBlocksByRootRequest { + request: BlocksByRootSingleRequest, + resolved: bool, +} + +impl ActiveBlocksByRootRequest { + pub fn new(request: BlocksByRootSingleRequest) -> Self { + Self { + request, + resolved: false, + } + } + + /// Append a response to the single chunk request. If the chunk is valid, the request is + /// resolved immediately. + /// The active request SHOULD be dropped after `add_response` returns an error + pub fn add_response( + &mut self, + block: Arc>, + ) -> Result>, LookupVerifyError> { + if self.resolved { + return Err(LookupVerifyError::TooManyResponses); + } + + let block_root = get_block_root(&block); + if self.request.0 != block_root { + return Err(LookupVerifyError::UnrequestedBlockRoot(block_root)); + } + + // Valid data, blocks by root expects a single response + self.resolved = true; + Ok(block) + } + + pub fn terminate(self) -> Result<(), LookupVerifyError> { + if self.resolved { + Ok(()) + } else { + Err(LookupVerifyError::NoResponseReturned) + } + } +} + +#[derive(Debug, Copy, Clone)] +pub struct BlocksByRootSingleRequest(pub Hash256); + +impl BlocksByRootSingleRequest { + pub fn into_request(self, spec: &ChainSpec) -> BlocksByRootRequest { + BlocksByRootRequest::new(vec![self.0], spec) + } +} + +#[derive(Debug, Clone)] +pub struct BlobsByRootSingleBlockRequest { + pub block_root: Hash256, + pub indices: Vec, +} + +impl BlobsByRootSingleBlockRequest { + pub fn into_request(self, spec: &ChainSpec) -> BlobsByRootRequest { + BlobsByRootRequest::new( + self.indices + .into_iter() + .map(|index| BlobIdentifier { + block_root: self.block_root, + index, + }) + .collect(), + spec, + ) + } +} + +pub struct ActiveBlobsByRootRequest { + request: BlobsByRootSingleBlockRequest, + blobs: Vec>>, + resolved: bool, +} + +impl ActiveBlobsByRootRequest { + pub fn new(request: BlobsByRootSingleBlockRequest) -> Self { + Self { + request, + blobs: vec![], + resolved: false, + } + } + + /// Appends a chunk to this multi-item request. If all expected chunks are received, this + /// method returns `Some`, resolving the request before the stream terminator. + /// The active request SHOULD be dropped after `add_response` returns an error + pub fn add_response( + &mut self, + blob: Arc>, + ) -> Result>>>, LookupVerifyError> { + if self.resolved { + return Err(LookupVerifyError::TooManyResponses); + } + + let block_root = blob.block_root(); + if self.request.block_root != block_root { + return Err(LookupVerifyError::UnrequestedBlockRoot(block_root)); + } + if !blob.verify_blob_sidecar_inclusion_proof().unwrap_or(false) { + return Err(LookupVerifyError::InvalidInclusionProof); + } + if !self.request.indices.contains(&blob.index) { + return Err(LookupVerifyError::UnrequestedBlobIndex(blob.index)); + } + if self.blobs.iter().any(|b| b.index == blob.index) { + return Err(LookupVerifyError::DuplicateData); + } + + self.blobs.push(blob); + if self.blobs.len() >= self.request.indices.len() { + // All expected chunks received, return result early + self.resolved = true; + Ok(Some(std::mem::take(&mut self.blobs))) + } else { + Ok(None) + } + } + + pub fn terminate(self) -> Option>>> { + if self.resolved { + None + } else { + Some(self.blobs) + } + } +} From 532206e008d47a702e4a699bcf0c26041cf72cb6 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Tue, 23 Apr 2024 02:06:46 +1000 Subject: [PATCH 17/37] Fix stuck backfill when scheduled work queue is at capacity (#5575) * Fix stuck backfill and add regression test. * Remove unnecessary `yield_now` * Merge branch 'unstable' into fix-stuck-backfill * Revert previous change and add extra comment. * Merge branch 'unstable' into fix-stuck-backfill * Update tests to use configured event schedule instead of hard coded values. * Merge branch 'unstable' of https://github.com/sigp/lighthouse into fix-stuck-backfill --- beacon_node/beacon_processor/Cargo.toml | 5 +- beacon_node/beacon_processor/src/lib.rs | 2 +- .../src/work_reprocessing_queue.rs | 163 ++++++++++++++---- common/slot_clock/src/manual_slot_clock.rs | 6 + 4 files changed, 142 insertions(+), 34 deletions(-) diff --git a/beacon_node/beacon_processor/Cargo.toml b/beacon_node/beacon_processor/Cargo.toml index 723b09b581c..6c49a28ec87 100644 --- a/beacon_node/beacon_processor/Cargo.toml +++ b/beacon_node/beacon_processor/Cargo.toml @@ -23,4 +23,7 @@ lazy_static = { workspace = true } lighthouse_metrics = { workspace = true } parking_lot = { workspace = true } num_cpus = { workspace = true } -serde = { workspace = true } \ No newline at end of file +serde = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index 9b83e9cacbc..fee55b39adc 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -851,7 +851,7 @@ impl BeaconProcessor { ready_work_tx, work_reprocessing_rx, &self.executor, - slot_clock, + Arc::new(slot_clock), self.log.clone(), maximum_gossip_clock_disparity, )?; diff --git a/beacon_node/beacon_processor/src/work_reprocessing_queue.rs b/beacon_node/beacon_processor/src/work_reprocessing_queue.rs index c9be28444c8..496fa683d2c 100644 --- a/beacon_node/beacon_processor/src/work_reprocessing_queue.rs +++ b/beacon_node/beacon_processor/src/work_reprocessing_queue.rs @@ -22,6 +22,7 @@ use slot_clock::SlotClock; use std::collections::{HashMap, HashSet}; use std::future::Future; use std::pin::Pin; +use std::sync::Arc; use std::task::Context; use std::time::Duration; use strum::AsRefStr; @@ -243,7 +244,7 @@ struct ReprocessQueue { attestation_delay_debounce: TimeLatch, lc_update_delay_debounce: TimeLatch, next_backfill_batch_event: Option>>, - slot_clock: Pin>, + slot_clock: Arc, } pub type QueuedLightClientUpdateId = usize; @@ -362,7 +363,7 @@ pub fn spawn_reprocess_scheduler( ready_work_tx: Sender, work_reprocessing_rx: Receiver, executor: &TaskExecutor, - slot_clock: S, + slot_clock: Arc, log: Logger, maximum_gossip_clock_disparity: Duration, ) -> Result<(), String> { @@ -370,34 +371,12 @@ pub fn spawn_reprocess_scheduler( if ADDITIONAL_QUEUED_BLOCK_DELAY >= maximum_gossip_clock_disparity { return Err("The block delay and gossip disparity don't match.".to_string()); } - let mut queue = ReprocessQueue { - work_reprocessing_rx, - ready_work_tx, - gossip_block_delay_queue: DelayQueue::new(), - rpc_block_delay_queue: DelayQueue::new(), - attestations_delay_queue: DelayQueue::new(), - lc_updates_delay_queue: DelayQueue::new(), - queued_gossip_block_roots: HashSet::new(), - queued_lc_updates: FnvHashMap::default(), - queued_aggregates: FnvHashMap::default(), - queued_unaggregates: FnvHashMap::default(), - awaiting_attestations_per_root: HashMap::new(), - awaiting_lc_updates_per_parent_root: HashMap::new(), - queued_backfill_batches: Vec::new(), - next_attestation: 0, - next_lc_update: 0, - early_block_debounce: TimeLatch::default(), - rpc_block_debounce: TimeLatch::default(), - attestation_delay_debounce: TimeLatch::default(), - lc_update_delay_debounce: TimeLatch::default(), - next_backfill_batch_event: None, - slot_clock: Box::pin(slot_clock.clone()), - }; + let mut queue = ReprocessQueue::new(ready_work_tx, work_reprocessing_rx, slot_clock); executor.spawn( async move { while let Some(msg) = queue.next().await { - queue.handle_message(msg, &slot_clock, &log); + queue.handle_message(msg, &log); } debug!( @@ -412,7 +391,37 @@ pub fn spawn_reprocess_scheduler( } impl ReprocessQueue { - fn handle_message(&mut self, msg: InboundEvent, slot_clock: &S, log: &Logger) { + fn new( + ready_work_tx: Sender, + work_reprocessing_rx: Receiver, + slot_clock: Arc, + ) -> Self { + ReprocessQueue { + work_reprocessing_rx, + ready_work_tx, + gossip_block_delay_queue: DelayQueue::new(), + rpc_block_delay_queue: DelayQueue::new(), + attestations_delay_queue: DelayQueue::new(), + lc_updates_delay_queue: DelayQueue::new(), + queued_gossip_block_roots: HashSet::new(), + queued_lc_updates: FnvHashMap::default(), + queued_aggregates: FnvHashMap::default(), + queued_unaggregates: FnvHashMap::default(), + awaiting_attestations_per_root: HashMap::new(), + awaiting_lc_updates_per_parent_root: HashMap::new(), + queued_backfill_batches: Vec::new(), + next_attestation: 0, + next_lc_update: 0, + early_block_debounce: TimeLatch::default(), + rpc_block_debounce: TimeLatch::default(), + attestation_delay_debounce: TimeLatch::default(), + lc_update_delay_debounce: TimeLatch::default(), + next_backfill_batch_event: None, + slot_clock, + } + } + + fn handle_message(&mut self, msg: InboundEvent, log: &Logger) { use ReprocessQueueMessage::*; match msg { // Some block has been indicated as "early" and should be processed when the @@ -426,7 +435,7 @@ impl ReprocessQueue { return; } - if let Some(duration_till_slot) = slot_clock.duration_to_slot(block_slot) { + if let Some(duration_till_slot) = self.slot_clock.duration_to_slot(block_slot) { // Check to ensure this won't over-fill the queue. if self.queued_gossip_block_roots.len() >= MAXIMUM_QUEUED_BLOCKS { if self.early_block_debounce.elapsed() { @@ -459,7 +468,7 @@ impl ReprocessQueue { // This logic is slightly awkward since `SlotClock::duration_to_slot` // doesn't distinguish between a slot that has already arrived and an // error reading the slot clock. - if let Some(now) = slot_clock.now() { + if let Some(now) = self.slot_clock.now() { if block_slot <= now && self .ready_work_tx @@ -860,7 +869,8 @@ impl ReprocessQueue { } } InboundEvent::ReadyBackfillSync(queued_backfill_batch) => { - let millis_from_slot_start = slot_clock + let millis_from_slot_start = self + .slot_clock .millis_from_current_slot_start() .map_or("null".to_string(), |duration| { duration.as_millis().to_string() @@ -886,7 +896,12 @@ impl ReprocessQueue { "Failed to send scheduled backfill work"; "info" => "sending work back to queue" ); - self.queued_backfill_batches.insert(0, batch) + self.queued_backfill_batches.insert(0, batch); + + // only recompute if there is no `next_backfill_batch_event` already scheduled + if self.next_backfill_batch_event.is_none() { + self.recompute_next_backfill_batch_event(); + } } // The message was not sent and we didn't get the correct // return result. This is a logic error. @@ -963,7 +978,11 @@ impl ReprocessQueue { #[cfg(test)] mod tests { use super::*; - use slot_clock::TestingSlotClock; + use logging::test_logger; + use slot_clock::{ManualSlotClock, TestingSlotClock}; + use std::ops::Add; + use std::sync::Arc; + use task_executor::test_utils::TestRuntime; #[test] fn backfill_processing_schedule_calculation() { @@ -1002,4 +1021,84 @@ mod tests { duration_to_next_slot + event_times[0] ); } + + // Regression test for issue #5504. + // See: https://github.com/sigp/lighthouse/issues/5504#issuecomment-2050930045 + #[tokio::test] + async fn backfill_schedule_failed_should_reschedule() { + let runtime = TestRuntime::default(); + let log = test_logger(); + let (work_reprocessing_tx, work_reprocessing_rx) = mpsc::channel(1); + let (ready_work_tx, mut ready_work_rx) = mpsc::channel(1); + let slot_duration = 12; + let slot_clock = Arc::new(testing_slot_clock(slot_duration)); + + spawn_reprocess_scheduler( + ready_work_tx.clone(), + work_reprocessing_rx, + &runtime.task_executor, + slot_clock.clone(), + log, + Duration::from_millis(500), + ) + .unwrap(); + + // Pause time so it only advances manually + tokio::time::pause(); + + // Send some random work to `ready_work_tx` to fill up the capacity first. + ready_work_tx + .try_send(ReadyWork::IgnoredRpcBlock(IgnoredRpcBlock { + process_fn: Box::new(|| {}), + })) + .unwrap(); + + // Now queue a backfill sync batch. + work_reprocessing_tx + .try_send(ReprocessQueueMessage::BackfillSync(QueuedBackfillBatch( + Box::pin(async {}), + ))) + .unwrap(); + tokio::task::yield_now().await; + + // Advance the time by more than 1/2 the slot to trigger a scheduled backfill batch to be sent. + // This should fail as the `ready_work` channel is at capacity, and it should be rescheduled. + let duration_to_next_event = + ReprocessQueue::duration_until_next_backfill_batch_event(slot_clock.as_ref()); + let one_ms = Duration::from_millis(1); + advance_time(&slot_clock, duration_to_next_event.add(one_ms)).await; + + // Now drain the `ready_work` channel. + assert!(matches!( + ready_work_rx.try_recv(), + Ok(ReadyWork::IgnoredRpcBlock { .. }) + )); + assert!(ready_work_rx.try_recv().is_err()); + + // Advance time again, and assert that the re-scheduled batch is successfully sent. + let duration_to_next_event = + ReprocessQueue::duration_until_next_backfill_batch_event(slot_clock.as_ref()); + advance_time(&slot_clock, duration_to_next_event.add(one_ms)).await; + assert!(matches!( + ready_work_rx.try_recv(), + Ok(ReadyWork::BackfillSync { .. }) + )); + } + + /// Advances slot clock and test clock time by the same duration. + async fn advance_time(slot_clock: &ManualSlotClock, duration: Duration) { + slot_clock.advance_time(duration); + tokio::time::advance(duration).await; + // NOTE: The `tokio::time::advance` fn actually calls `yield_now()` after advancing the + // clock. Why do we need an extra `yield_now`? + tokio::task::yield_now().await; + } + + fn testing_slot_clock(slot_duration: u64) -> ManualSlotClock { + TestingSlotClock::new( + Slot::new(0), + Duration::from_secs(0), + Duration::from_secs(slot_duration), + ) + } } diff --git a/common/slot_clock/src/manual_slot_clock.rs b/common/slot_clock/src/manual_slot_clock.rs index 7b42fa9062d..1d71533de15 100644 --- a/common/slot_clock/src/manual_slot_clock.rs +++ b/common/slot_clock/src/manual_slot_clock.rs @@ -1,5 +1,6 @@ use super::SlotClock; use parking_lot::RwLock; +use std::ops::Add; use std::sync::Arc; use std::time::Duration; use types::Slot; @@ -41,6 +42,11 @@ impl ManualSlotClock { *self.current_time.write() = duration; } + pub fn advance_time(&self, duration: Duration) { + let current_time = *self.current_time.read(); + *self.current_time.write() = current_time.add(duration); + } + pub fn advance_slot(&self) { self.set_slot(self.now().unwrap().as_u64() + 1) } From ad7f0e0cdb471749d21ce978383e18c8db6ad4bd Mon Sep 17 00:00:00 2001 From: chonghe <44791194+chong-he@users.noreply.github.com> Date: Tue, 23 Apr 2024 00:07:43 +0800 Subject: [PATCH 18/37] Delete repetitive execute command in local testnet scripts (#5611) * Delete repetitive execute * Merge branch 'unstable' of https://github.com/sigp/lighthouse into local-testnet --- scripts/local_testnet/start_local_testnet.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/local_testnet/start_local_testnet.sh b/scripts/local_testnet/start_local_testnet.sh index 77422095130..be91d069985 100755 --- a/scripts/local_testnet/start_local_testnet.sh +++ b/scripts/local_testnet/start_local_testnet.sh @@ -112,9 +112,6 @@ sleeping 3 execute_command_add_PID el_bootnode.log ./el_bootnode.sh sleeping 3 -execute_command_add_PID el_bootnode.log ./el_bootnode.sh -sleeping 1 - # Start beacon nodes BN_udp_tcp_base=9000 BN_http_port_base=8000 From 82b131d37fef84b93c4a3477ee8093a0b39f81e7 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Mon, 22 Apr 2024 19:21:21 -0500 Subject: [PATCH 19/37] Electra: Add New Containers (#5607) * Electra: Add New Containers --- consensus/types/src/consolidation.rs | 35 ++++++++++++++++++ consensus/types/src/deposit_receipt.rs | 37 +++++++++++++++++++ .../src/execution_layer_withdrawal_request.rs | 34 +++++++++++++++++ consensus/types/src/lib.rs | 14 +++++++ .../types/src/pending_balance_deposit.rs | 33 +++++++++++++++++ consensus/types/src/pending_consolidation.rs | 33 +++++++++++++++++ .../types/src/pending_partial_withdrawal.rs | 35 ++++++++++++++++++ consensus/types/src/signed_consolidation.rs | 32 ++++++++++++++++ 8 files changed, 253 insertions(+) create mode 100644 consensus/types/src/consolidation.rs create mode 100644 consensus/types/src/deposit_receipt.rs create mode 100644 consensus/types/src/execution_layer_withdrawal_request.rs create mode 100644 consensus/types/src/pending_balance_deposit.rs create mode 100644 consensus/types/src/pending_consolidation.rs create mode 100644 consensus/types/src/pending_partial_withdrawal.rs create mode 100644 consensus/types/src/signed_consolidation.rs diff --git a/consensus/types/src/consolidation.rs b/consensus/types/src/consolidation.rs new file mode 100644 index 00000000000..09a2d4bb0c3 --- /dev/null +++ b/consensus/types/src/consolidation.rs @@ -0,0 +1,35 @@ +use crate::test_utils::TestRandom; +use crate::Epoch; +use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; + +#[derive( + arbitrary::Arbitrary, + Debug, + PartialEq, + Eq, + Hash, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, +)] +pub struct Consolidation { + #[serde(with = "serde_utils::quoted_u64")] + pub source_index: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub target_index: u64, + pub epoch: Epoch, +} + +#[cfg(test)] +mod tests { + use super::*; + + ssz_and_tree_hash_tests!(Consolidation); +} diff --git a/consensus/types/src/deposit_receipt.rs b/consensus/types/src/deposit_receipt.rs new file mode 100644 index 00000000000..6a08f717f3d --- /dev/null +++ b/consensus/types/src/deposit_receipt.rs @@ -0,0 +1,37 @@ +use crate::test_utils::TestRandom; +use crate::{Hash256, PublicKeyBytes, Signature}; +use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; + +#[derive( + arbitrary::Arbitrary, + Debug, + PartialEq, + Eq, + Hash, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, +)] +pub struct DepositReceipt { + pub pubkey: PublicKeyBytes, + pub withdrawal_credentials: Hash256, + #[serde(with = "serde_utils::quoted_u64")] + pub amount: u64, + pub signature: Signature, + #[serde(with = "serde_utils::quoted_u64")] + pub index: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + ssz_and_tree_hash_tests!(DepositReceipt); +} diff --git a/consensus/types/src/execution_layer_withdrawal_request.rs b/consensus/types/src/execution_layer_withdrawal_request.rs new file mode 100644 index 00000000000..b1d814c2834 --- /dev/null +++ b/consensus/types/src/execution_layer_withdrawal_request.rs @@ -0,0 +1,34 @@ +use crate::test_utils::TestRandom; +use crate::{Address, PublicKeyBytes}; +use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; + +#[derive( + arbitrary::Arbitrary, + Debug, + PartialEq, + Eq, + Hash, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, +)] +pub struct ExecutionLayerWithdrawalRequest { + pub source_address: Address, + pub validator_pubkey: PublicKeyBytes, + #[serde(with = "serde_utils::quoted_u64")] + pub amount: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + ssz_and_tree_hash_tests!(ExecutionLayerWithdrawalRequest); +} diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 6551ebc1dda..dee55789398 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -29,16 +29,19 @@ pub mod bls_to_execution_change; pub mod builder_bid; pub mod chain_spec; pub mod checkpoint; +pub mod consolidation; pub mod consts; pub mod contribution_and_proof; pub mod deposit; pub mod deposit_data; pub mod deposit_message; +pub mod deposit_receipt; pub mod deposit_tree_snapshot; pub mod enr_fork_id; pub mod eth1_data; pub mod eth_spec; pub mod execution_block_hash; +pub mod execution_layer_withdrawal_request; pub mod execution_payload; pub mod execution_payload_header; pub mod fork; @@ -54,6 +57,9 @@ pub mod light_client_finality_update; pub mod light_client_optimistic_update; pub mod light_client_update; pub mod pending_attestation; +pub mod pending_balance_deposit; +pub mod pending_consolidation; +pub mod pending_partial_withdrawal; pub mod proposer_preparation_data; pub mod proposer_slashing; pub mod relative_epoch; @@ -63,6 +69,7 @@ pub mod signed_aggregate_and_proof; pub mod signed_beacon_block; pub mod signed_beacon_block_header; pub mod signed_bls_to_execution_change; +pub mod signed_consolidation; pub mod signed_contribution_and_proof; pub mod signed_voluntary_exit; pub mod signing_data; @@ -133,10 +140,12 @@ pub use crate::checkpoint::Checkpoint; pub use crate::config_and_preset::{ ConfigAndPreset, ConfigAndPresetCapella, ConfigAndPresetDeneb, ConfigAndPresetElectra, }; +pub use crate::consolidation::Consolidation; pub use crate::contribution_and_proof::ContributionAndProof; pub use crate::deposit::{Deposit, DEPOSIT_TREE_DEPTH}; pub use crate::deposit_data::DepositData; pub use crate::deposit_message::DepositMessage; +pub use crate::deposit_receipt::DepositReceipt; pub use crate::deposit_tree_snapshot::{DepositTreeSnapshot, FinalizedExecutionBlock}; pub use crate::enr_fork_id::EnrForkId; pub use crate::epoch_cache::{EpochCache, EpochCacheError, EpochCacheKey}; @@ -144,6 +153,7 @@ pub use crate::eth1_data::Eth1Data; pub use crate::eth_spec::EthSpecId; pub use crate::execution_block_hash::ExecutionBlockHash; pub use crate::execution_block_header::ExecutionBlockHeader; +pub use crate::execution_layer_withdrawal_request::ExecutionLayerWithdrawalRequest; pub use crate::execution_payload::{ ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadElectra, ExecutionPayloadMerge, ExecutionPayloadRef, Transaction, Transactions, Withdrawals, @@ -189,6 +199,9 @@ pub use crate::payload::{ FullPayloadRef, OwnedExecPayload, }; pub use crate::pending_attestation::PendingAttestation; +pub use crate::pending_balance_deposit::PendingBalanceDeposit; +pub use crate::pending_consolidation::PendingConsolidation; +pub use crate::pending_partial_withdrawal::PendingPartialWithdrawal; pub use crate::preset::{ AltairPreset, BasePreset, BellatrixPreset, CapellaPreset, DenebPreset, ElectraPreset, }; @@ -207,6 +220,7 @@ pub use crate::signed_beacon_block::{ }; pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; pub use crate::signed_bls_to_execution_change::SignedBlsToExecutionChange; +pub use crate::signed_consolidation::SignedConsolidation; pub use crate::signed_contribution_and_proof::SignedContributionAndProof; pub use crate::signed_voluntary_exit::SignedVoluntaryExit; pub use crate::signing_data::{SignedRoot, SigningData}; diff --git a/consensus/types/src/pending_balance_deposit.rs b/consensus/types/src/pending_balance_deposit.rs new file mode 100644 index 00000000000..a2bce577f87 --- /dev/null +++ b/consensus/types/src/pending_balance_deposit.rs @@ -0,0 +1,33 @@ +use crate::test_utils::TestRandom; +use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; + +#[derive( + arbitrary::Arbitrary, + Debug, + PartialEq, + Eq, + Hash, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, +)] +pub struct PendingBalanceDeposit { + #[serde(with = "serde_utils::quoted_u64")] + pub index: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub amount: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + ssz_and_tree_hash_tests!(PendingBalanceDeposit); +} diff --git a/consensus/types/src/pending_consolidation.rs b/consensus/types/src/pending_consolidation.rs new file mode 100644 index 00000000000..6e0b74a7383 --- /dev/null +++ b/consensus/types/src/pending_consolidation.rs @@ -0,0 +1,33 @@ +use crate::test_utils::TestRandom; +use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; + +#[derive( + arbitrary::Arbitrary, + Debug, + PartialEq, + Eq, + Hash, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, +)] +pub struct PendingConsolidation { + #[serde(with = "serde_utils::quoted_u64")] + pub source_index: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub target_index: u64, +} + +#[cfg(test)] +mod tests { + use super::*; + + ssz_and_tree_hash_tests!(PendingConsolidation); +} diff --git a/consensus/types/src/pending_partial_withdrawal.rs b/consensus/types/src/pending_partial_withdrawal.rs new file mode 100644 index 00000000000..e5ace7b2736 --- /dev/null +++ b/consensus/types/src/pending_partial_withdrawal.rs @@ -0,0 +1,35 @@ +use crate::test_utils::TestRandom; +use crate::Epoch; +use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; + +#[derive( + arbitrary::Arbitrary, + Debug, + PartialEq, + Eq, + Hash, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, +)] +pub struct PendingPartialWithdrawal { + #[serde(with = "serde_utils::quoted_u64")] + pub index: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub amount: u64, + pub withdrawable_epoch: Epoch, +} + +#[cfg(test)] +mod tests { + use super::*; + + ssz_and_tree_hash_tests!(PendingPartialWithdrawal); +} diff --git a/consensus/types/src/signed_consolidation.rs b/consensus/types/src/signed_consolidation.rs new file mode 100644 index 00000000000..f004ec23bd4 --- /dev/null +++ b/consensus/types/src/signed_consolidation.rs @@ -0,0 +1,32 @@ +use crate::test_utils::TestRandom; +use crate::{Consolidation, Signature}; +use serde::{Deserialize, Serialize}; +use ssz_derive::{Decode, Encode}; +use test_random_derive::TestRandom; +use tree_hash_derive::TreeHash; + +#[derive( + arbitrary::Arbitrary, + Debug, + PartialEq, + Eq, + Hash, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + TestRandom, +)] +pub struct SignedConsolidation { + pub message: Consolidation, + pub signature: Signature, +} + +#[cfg(test)] +mod tests { + use super::*; + + ssz_and_tree_hash_tests!(SignedConsolidation); +} From 72a33604b3f1bd5ba9e23812ae53b1df4f29e405 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 23 Apr 2024 23:13:34 +1000 Subject: [PATCH 20/37] Add timing for block availability (#5510) * Add timing for block availability * Attestation metrics analysis * Prettier printing * Add some metrics and timings to track late blocks * Update to latest unstable * fmt * Merge latest unstable * Small tweaks * Try pushing blob timing down into verification * Simplify for clippy --- beacon_node/beacon_chain/src/beacon_chain.rs | 65 +++++----- .../beacon_chain/src/blob_verification.rs | 42 +++++-- .../beacon_chain/src/block_times_cache.rs | 89 +++++++++++++- .../beacon_chain/src/block_verification.rs | 3 +- .../beacon_chain/src/canonical_head.rs | 115 +++++++++++++++--- .../src/data_availability_checker.rs | 23 +++- .../src/data_availability_checker/error.rs | 4 +- .../overflow_lru_cache.rs | 7 ++ beacon_node/beacon_chain/src/metrics.rs | 72 ++++++----- beacon_node/execution_layer/src/lib.rs | 24 ++-- beacon_node/network/src/metrics.rs | 63 ++++------ .../gossip_methods.rs | 40 +++--- 12 files changed, 391 insertions(+), 156 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index b3790024f81..c59c5e8ed10 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2953,7 +2953,7 @@ impl BeaconChain { } /// Wraps `process_block` in logic to cache the block's commitments in the processing cache - /// and evict if the block was imported or erred. + /// and evict if the block was imported or errored. pub async fn process_block_with_early_caching>( self: &Arc, block_root: Hash256, @@ -2998,22 +2998,20 @@ impl BeaconChain { // Increment the Prometheus counter for block processing requests. metrics::inc_counter(&metrics::BLOCK_PROCESSING_REQUESTS); + let block_slot = unverified_block.block().slot(); + // Set observed time if not already set. Usually this should be set by gossip or RPC, // but just in case we set it again here (useful for tests). - if let (Some(seen_timestamp), Some(current_slot)) = - (self.slot_clock.now_duration(), self.slot_clock.now()) - { + if let Some(seen_timestamp) = self.slot_clock.now_duration() { self.block_times_cache.write().set_time_observed( block_root, - current_slot, + block_slot, seen_timestamp, None, None, ); } - let block_slot = unverified_block.block().slot(); - // A small closure to group the verification and import errors. let chain = self.clone(); let import_block = async move { @@ -3024,6 +3022,15 @@ impl BeaconChain { )?; publish_fn()?; let executed_block = chain.into_executed_block(execution_pending).await?; + // Record the time it took to ask the execution layer. + if let Some(seen_timestamp) = self.slot_clock.now_duration() { + self.block_times_cache.write().set_execution_time( + block_root, + block_slot, + seen_timestamp, + ) + } + match executed_block { ExecutedBlock::Available(block) => { self.import_available_block(Box::new(block)).await @@ -3090,8 +3097,8 @@ impl BeaconChain { } } - /// Accepts a fully-verified block and awaits on it's payload verification handle to - /// get a fully `ExecutedBlock` + /// Accepts a fully-verified block and awaits on its payload verification handle to + /// get a fully `ExecutedBlock`. /// /// An error is returned if the verification handle couldn't be awaited. pub async fn into_executed_block( @@ -3224,10 +3231,6 @@ impl BeaconChain { ) -> Result> { match availability { Availability::Available(block) => { - // This is the time since start of the slot where all the components of the block have become available - let delay = - get_slot_delay_ms(timestamp_now(), block.block.slot(), &self.slot_clock); - metrics::observe_duration(&metrics::BLOCK_AVAILABILITY_DELAY, delay); // Block is fully available, import into fork choice self.import_available_block(block).await } @@ -3256,6 +3259,15 @@ impl BeaconChain { consensus_context, } = import_data; + // Record the time at which this block's blobs became available. + if let Some(blobs_available) = block.blobs_available_timestamp() { + self.block_times_cache.write().set_time_blob_observed( + block_root, + block.slot(), + blobs_available, + ); + } + // import let chain = self.clone(); let block_root = self @@ -3396,6 +3408,14 @@ impl BeaconChain { "Early attester cache insert failed"; "error" => ?e ); + } else { + let attestable_timestamp = + self.slot_clock.now_duration().unwrap_or_default(); + self.block_times_cache.write().set_time_attestable( + block_root, + signed_block.slot(), + attestable_timestamp, + ) } } else { warn!( @@ -3885,25 +3905,6 @@ impl BeaconChain { ); } - // Do not store metrics if the block was > 4 slots old, this helps prevent noise during - // sync. - if block_delay_total < self.slot_clock.slot_duration() * 4 { - // Observe the delay between when we observed the block and when we imported it. - let block_delays = self.block_times_cache.read().get_block_delays( - block_root, - self.slot_clock - .start_of(current_slot) - .unwrap_or_else(|| Duration::from_secs(0)), - ); - - metrics::observe_duration( - &metrics::BEACON_BLOCK_IMPORTED_OBSERVED_DELAY_TIME, - block_delays - .imported - .unwrap_or_else(|| Duration::from_secs(0)), - ); - } - if let Some(event_handler) = self.event_handler.as_ref() { if event_handler.has_block_subscribers() { event_handler.register(EventKind::Block(SseBlock { diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 1fb61702006..a1ae260d930 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -14,6 +14,7 @@ use merkle_proof::MerkleTreeError; use slog::{debug, warn}; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; +use std::time::Duration; use tree_hash::TreeHash; use types::blob_sidecar::BlobIdentifier; use types::{ @@ -214,7 +215,10 @@ impl GossipVerifiedBlob { pub fn __assumed_valid(blob: Arc>) -> Self { Self { block_root: blob.block_root(), - blob: KzgVerifiedBlob { blob }, + blob: KzgVerifiedBlob { + blob, + seen_timestamp: Duration::from_secs(0), + }, } } pub fn id(&self) -> BlobIdentifier { @@ -260,6 +264,8 @@ impl GossipVerifiedBlob { #[ssz(struct_behaviour = "transparent")] pub struct KzgVerifiedBlob { blob: Arc>, + #[ssz(skip_serializing, skip_deserializing)] + seen_timestamp: Duration, } impl PartialOrd for KzgVerifiedBlob { @@ -275,8 +281,12 @@ impl Ord for KzgVerifiedBlob { } impl KzgVerifiedBlob { - pub fn new(blob: Arc>, kzg: &Kzg) -> Result { - verify_kzg_for_blob(blob, kzg) + pub fn new( + blob: Arc>, + kzg: &Kzg, + seen_timestamp: Duration, + ) -> Result { + verify_kzg_for_blob(blob, kzg, seen_timestamp) } pub fn to_blob(self) -> Arc> { self.blob @@ -294,12 +304,18 @@ impl KzgVerifiedBlob { pub fn blob_index(&self) -> u64 { self.blob.index } + pub fn seen_timestamp(&self) -> Duration { + self.seen_timestamp + } /// Construct a `KzgVerifiedBlob` that is assumed to be valid. /// /// This should ONLY be used for testing. #[cfg(test)] pub fn __assumed_valid(blob: Arc>) -> Self { - Self { blob } + Self { + blob, + seen_timestamp: Duration::from_secs(0), + } } } @@ -309,9 +325,13 @@ impl KzgVerifiedBlob { pub fn verify_kzg_for_blob( blob: Arc>, kzg: &Kzg, + seen_timestamp: Duration, ) -> Result, KzgError> { validate_blob::(kzg, &blob.blob, blob.kzg_commitment, blob.kzg_proof)?; - Ok(KzgVerifiedBlob { blob }) + Ok(KzgVerifiedBlob { + blob, + seen_timestamp, + }) } pub struct KzgVerifiedBlobList { @@ -322,13 +342,17 @@ impl KzgVerifiedBlobList { pub fn new>>>( blob_list: I, kzg: &Kzg, + seen_timestamp: Duration, ) -> Result { let blobs = blob_list.into_iter().collect::>(); verify_kzg_for_blob_list(blobs.iter(), kzg)?; Ok(Self { verified_blobs: blobs .into_iter() - .map(|blob| KzgVerifiedBlob { blob }) + .map(|blob| KzgVerifiedBlob { + blob, + seen_timestamp, + }) .collect(), }) } @@ -374,6 +398,8 @@ pub fn validate_blob_sidecar_for_gossip( let blob_epoch = blob_slot.epoch(T::EthSpec::slots_per_epoch()); let signed_block_header = &blob_sidecar.signed_block_header; + let seen_timestamp = chain.slot_clock.now_duration().unwrap_or_default(); + // This condition is not possible if we have received the blob from the network // since we only subscribe to `MaxBlobsPerBlock` subnets over gossip network. // We include this check only for completeness. @@ -641,8 +667,8 @@ pub fn validate_blob_sidecar_for_gossip( .kzg .as_ref() .ok_or(GossipBlobError::KzgNotInitialized)?; - let kzg_verified_blob = - KzgVerifiedBlob::new(blob_sidecar, kzg).map_err(GossipBlobError::KzgError)?; + let kzg_verified_blob = KzgVerifiedBlob::new(blob_sidecar, kzg, seen_timestamp) + .map_err(GossipBlobError::KzgError)?; Ok(GossipVerifiedBlob { block_root, diff --git a/beacon_node/beacon_chain/src/block_times_cache.rs b/beacon_node/beacon_chain/src/block_times_cache.rs index c5293bcb0ee..db547a1186c 100644 --- a/beacon_node/beacon_chain/src/block_times_cache.rs +++ b/beacon_node/beacon_chain/src/block_times_cache.rs @@ -18,6 +18,9 @@ type BlockRoot = Hash256; #[derive(Clone, Default)] pub struct Timestamps { pub observed: Option, + pub all_blobs_observed: Option, + pub execution_time: Option, + pub attestable: Option, pub imported: Option, pub set_as_head: Option, } @@ -25,8 +28,25 @@ pub struct Timestamps { // Helps arrange delay data so it is more relevant to metrics. #[derive(Debug, Default)] pub struct BlockDelays { + /// Time after start of slot we saw the block. pub observed: Option, + /// The time after the start of the slot we saw all blobs. + pub all_blobs_observed: Option, + /// The time it took to get verification from the EL for the block. + pub execution_time: Option, + /// The delay from the start of the slot before the block became available + /// + /// Equal to max(`observed + execution_time`, `all_blobs_observed`). + pub available: Option, + /// Time after `available`. + pub attestable: Option, + /// Time + /// ALSO time after `available`. + /// + /// We need to use `available` again rather than `attestable` to handle the case where the block + /// does not get added to the early-attester cache. pub imported: Option, + /// Time after `imported`. pub set_as_head: Option, } @@ -35,14 +55,34 @@ impl BlockDelays { let observed = times .observed .and_then(|observed_time| observed_time.checked_sub(slot_start_time)); + let all_blobs_observed = times + .all_blobs_observed + .and_then(|all_blobs_observed| all_blobs_observed.checked_sub(slot_start_time)); + let execution_time = times + .execution_time + .and_then(|execution_time| execution_time.checked_sub(times.observed?)); + // Duration since UNIX epoch at which block became available. + let available_time = times.execution_time.map(|execution_time| { + std::cmp::max(execution_time, times.all_blobs_observed.unwrap_or_default()) + }); + // Duration from the start of the slot until the block became available. + let available_delay = + available_time.and_then(|available_time| available_time.checked_sub(slot_start_time)); + let attestable = times + .attestable + .and_then(|attestable_time| attestable_time.checked_sub(slot_start_time)); let imported = times .imported - .and_then(|imported_time| imported_time.checked_sub(times.observed?)); + .and_then(|imported_time| imported_time.checked_sub(available_time?)); let set_as_head = times .set_as_head .and_then(|set_as_head_time| set_as_head_time.checked_sub(times.imported?)); BlockDelays { observed, + all_blobs_observed, + execution_time, + available: available_delay, + attestable, imported, set_as_head, } @@ -109,6 +149,53 @@ impl BlockTimesCache { } } + pub fn set_time_blob_observed( + &mut self, + block_root: BlockRoot, + slot: Slot, + timestamp: Duration, + ) { + let block_times = self + .cache + .entry(block_root) + .or_insert_with(|| BlockTimesCacheValue::new(slot)); + if block_times + .timestamps + .all_blobs_observed + .map_or(true, |prev| timestamp > prev) + { + block_times.timestamps.all_blobs_observed = Some(timestamp); + } + } + + pub fn set_execution_time(&mut self, block_root: BlockRoot, slot: Slot, timestamp: Duration) { + let block_times = self + .cache + .entry(block_root) + .or_insert_with(|| BlockTimesCacheValue::new(slot)); + if block_times + .timestamps + .execution_time + .map_or(true, |prev| timestamp < prev) + { + block_times.timestamps.execution_time = Some(timestamp); + } + } + + pub fn set_time_attestable(&mut self, block_root: BlockRoot, slot: Slot, timestamp: Duration) { + let block_times = self + .cache + .entry(block_root) + .or_insert_with(|| BlockTimesCacheValue::new(slot)); + if block_times + .timestamps + .attestable + .map_or(true, |prev| timestamp < prev) + { + block_times.timestamps.attestable = Some(timestamp); + } + } + pub fn set_time_imported(&mut self, block_root: BlockRoot, slot: Slot, timestamp: Duration) { let block_times = self .cache diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 3cd8a7f259b..38648949f9c 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -666,8 +666,7 @@ type PayloadVerificationHandle = /// - Parent is known /// - Signatures /// - State root check -/// - Per block processing -/// - Blobs sidecar has been validated if present +/// - Block processing /// /// Note: a `ExecutionPendingBlock` is not _forever_ valid to be imported, it may later become invalid /// due to finality or some other event. A `ExecutionPendingBlock` should be imported into the diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index ced4eda05cf..734575d2c0d 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -1405,13 +1405,6 @@ fn observe_head_block_delays( // Do not store metrics if the block was > 4 slots old, this helps prevent noise during // sync. if !block_from_sync { - // Observe the total block delay. This is the delay between the time the slot started - // and when the block was set as head. - metrics::observe_duration( - &metrics::BEACON_BLOCK_HEAD_SLOT_START_DELAY_TIME, - block_delay_total, - ); - // Observe the delay between when we imported the block and when we set the block as // head. let block_delays = block_times_cache.get_block_delays( @@ -1421,34 +1414,120 @@ fn observe_head_block_delays( .unwrap_or_else(|| Duration::from_secs(0)), ); - metrics::observe_duration( - &metrics::BEACON_BLOCK_OBSERVED_SLOT_START_DELAY_TIME, + // Update all the metrics + + // Convention here is to use "Time" to indicate the duration of the event and "Delay" + // to indicate the time since the start of the slot. + // + // Observe the total block delay. This is the delay between the time the slot started + // and when the block was set as head. + metrics::set_gauge( + &metrics::BEACON_BLOCK_DELAY_TOTAL, + block_delay_total.as_millis() as i64, + ); + + // The time at which the beacon block was first observed to be processed + metrics::set_gauge( + &metrics::BEACON_BLOCK_DELAY_OBSERVED_SLOT_START, block_delays .observed - .unwrap_or_else(|| Duration::from_secs(0)), + .unwrap_or_else(|| Duration::from_secs(0)) + .as_millis() as i64, + ); + + // The time from the start of the slot when all blobs have been observed. Technically this + // is the time we last saw a blob related to this block/slot. + metrics::set_gauge( + &metrics::BEACON_BLOB_DELAY_ALL_OBSERVED_SLOT_START, + block_delays + .all_blobs_observed + .unwrap_or_else(|| Duration::from_secs(0)) + .as_millis() as i64, ); - metrics::observe_duration( - &metrics::BEACON_BLOCK_HEAD_IMPORTED_DELAY_TIME, + // The time it took to check the validity with the EL + metrics::set_gauge( + &metrics::BEACON_BLOCK_DELAY_EXECUTION_TIME, + block_delays + .execution_time + .unwrap_or_else(|| Duration::from_secs(0)) + .as_millis() as i64, + ); + + // The time the block became available after the start of the slot. Available here means + // that all the blobs have arrived and the block has been verified by the execution layer. + metrics::set_gauge( + &metrics::BEACON_BLOCK_DELAY_AVAILABLE_SLOT_START, + block_delays + .available + .unwrap_or_else(|| Duration::from_secs(0)) + .as_millis() as i64, + ); + + // The time the block became attestable after the start of the slot. + metrics::set_gauge( + &metrics::BEACON_BLOCK_DELAY_ATTESTABLE_SLOT_START, + block_delays + .attestable + .unwrap_or_else(|| Duration::from_secs(0)) + .as_millis() as i64, + ); + + // The time the block was imported since becoming available. + metrics::set_gauge( + &metrics::BEACON_BLOCK_DELAY_IMPORTED_TIME, + block_delays + .imported + .unwrap_or_else(|| Duration::from_secs(0)) + .as_millis() as i64, + ); + + // The time the block was imported and setting it as head + metrics::set_gauge( + &metrics::BEACON_BLOCK_DELAY_HEAD_IMPORTED_TIME, block_delays .set_as_head - .unwrap_or_else(|| Duration::from_secs(0)), + .unwrap_or_else(|| Duration::from_secs(0)) + .as_millis() as i64, ); // If the block was enshrined as head too late for attestations to be created for it, // log a debug warning and increment a metric. + let format_delay = |delay: &Option| { + delay.map_or("unknown".to_string(), |d| format!("{}", d.as_millis())) + }; if late_head { - metrics::inc_counter(&metrics::BEACON_BLOCK_HEAD_SLOT_START_DELAY_EXCEEDED_TOTAL); + metrics::inc_counter(&metrics::BEACON_BLOCK_DELAY_HEAD_SLOT_START_EXCEEDED_TOTAL); debug!( log, "Delayed head block"; "block_root" => ?head_block_root, "proposer_index" => head_block_proposer_index, "slot" => head_block_slot, - "block_delay" => ?block_delay_total, - "observed_delay" => ?block_delays.observed, - "imported_delay" => ?block_delays.imported, - "set_as_head_delay" => ?block_delays.set_as_head, + "total_delay_ms" => block_delay_total.as_millis(), + "observed_delay_ms" => format_delay(&block_delays.observed), + "blob_delay_ms" => format_delay(&block_delays.all_blobs_observed), + "execution_time_ms" => format_delay(&block_delays.execution_time), + "available_delay_ms" => format_delay(&block_delays.available), + "attestable_delay_ms" => format_delay(&block_delays.attestable), + "imported_time_ms" => format_delay(&block_delays.imported), + "set_as_head_time_ms" => format_delay(&block_delays.set_as_head), + ); + } else { + debug!( + log, + "On-time head block"; + "block_root" => ?head_block_root, + "proposer_index" => head_block_proposer_index, + "slot" => head_block_slot, + "total_delay_ms" => block_delay_total.as_millis(), + "observed_delay_ms" => format_delay(&block_delays.observed), + "blob_delay_ms" => format_delay(&block_delays.all_blobs_observed), + "execution_time_ms" => format_delay(&block_delays.execution_time), + "available_delay_ms" => format_delay(&block_delays.available), + "attestable_delay_ms" => format_delay(&block_delays.attestable), + "imported_time_ms" => format_delay(&block_delays.imported), + "set_as_head_time_ms" => format_delay(&block_delays.set_as_head), ); } } diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 3ef105c6d34..dd0d97b1dae 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -14,6 +14,7 @@ use std::fmt; use std::fmt::Debug; use std::num::NonZeroUsize; use std::sync::Arc; +use std::time::Duration; use task_executor::TaskExecutor; use types::blob_sidecar::{BlobIdentifier, BlobSidecar, FixedBlobSidecarList}; use types::{BlobSidecarList, ChainSpec, Epoch, EthSpec, Hash256, SignedBeaconBlock}; @@ -176,8 +177,14 @@ impl DataAvailabilityChecker { return Err(AvailabilityCheckError::KzgNotInitialized); }; - let verified_blobs = KzgVerifiedBlobList::new(Vec::from(blobs).into_iter().flatten(), kzg) - .map_err(AvailabilityCheckError::Kzg)?; + let seen_timestamp = self + .slot_clock + .now_duration() + .ok_or(AvailabilityCheckError::SlotClockError)?; + + let verified_blobs = + KzgVerifiedBlobList::new(Vec::from(blobs).into_iter().flatten(), kzg, seen_timestamp) + .map_err(AvailabilityCheckError::Kzg)?; self.availability_cache .put_kzg_verified_blobs(block_root, verified_blobs) @@ -225,6 +232,7 @@ impl DataAvailabilityChecker { block_root, block, blobs: None, + blobs_available_timestamp: None, })) } } @@ -244,6 +252,7 @@ impl DataAvailabilityChecker { block_root, block, blobs: verified_blobs, + blobs_available_timestamp: None, })) } } @@ -289,6 +298,7 @@ impl DataAvailabilityChecker { block_root, block, blobs: None, + blobs_available_timestamp: None, })) } } @@ -303,6 +313,7 @@ impl DataAvailabilityChecker { block_root, block, blobs: verified_blobs, + blobs_available_timestamp: None, })) } } @@ -462,6 +473,8 @@ pub struct AvailableBlock { block_root: Hash256, block: Arc>, blobs: Option>, + /// Timestamp at which this block first became available (UNIX timestamp, time since 1970). + blobs_available_timestamp: Option, } impl AvailableBlock { @@ -474,6 +487,7 @@ impl AvailableBlock { block_root, block, blobs, + blobs_available_timestamp: None, } } @@ -488,6 +502,10 @@ impl AvailableBlock { self.blobs.as_ref() } + pub fn blobs_available_timestamp(&self) -> Option { + self.blobs_available_timestamp + } + pub fn deconstruct( self, ) -> ( @@ -499,6 +517,7 @@ impl AvailableBlock { block_root, block, blobs, + blobs_available_timestamp: _, } = self; (block_root, block, blobs) } diff --git a/beacon_node/beacon_chain/src/data_availability_checker/error.rs b/beacon_node/beacon_chain/src/data_availability_checker/error.rs index 0804fe3b9ab..6c524786bfa 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/error.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/error.rs @@ -19,6 +19,7 @@ pub enum Error { ParentStateMissing(Hash256), BlockReplayError(state_processing::BlockReplayError), RebuildingStateCaches(BeaconStateError), + SlotClockError, } pub enum ErrorCategory { @@ -39,7 +40,8 @@ impl Error { | Error::Unexpected | Error::ParentStateMissing(_) | Error::BlockReplayError(_) - | Error::RebuildingStateCaches(_) => ErrorCategory::Internal, + | Error::RebuildingStateCaches(_) + | Error::SlotClockError => ErrorCategory::Internal, Error::Kzg(_) | Error::BlobIndexInvalid(_) | Error::KzgCommitmentMismatch { .. } diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index edd981e6ddb..f4c1bc308c0 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -204,6 +204,12 @@ impl PendingComponents { executed_block, } = self; + let blobs_available_timestamp = verified_blobs + .iter() + .flatten() + .map(|blob| blob.seen_timestamp()) + .max(); + let Some(diet_executed_block) = executed_block else { return Err(AvailabilityCheckError::Unexpected); }; @@ -231,6 +237,7 @@ impl PendingComponents { block_root, block, blobs: Some(verified_blobs), + blobs_available_timestamp, }; Ok(Availability::Available(Box::new( AvailableExecutedBlock::new(available_block, import_data, payload_verification_outcome), diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 6cb0b6fd766..df718413cc0 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -847,37 +847,55 @@ lazy_static! { "Number of attester slashings seen", &["src", "validator"] ); +} + +// Prevent recursion limit +lazy_static! { /* * Block Delay Metrics */ - pub static ref BEACON_BLOCK_OBSERVED_SLOT_START_DELAY_TIME: Result = try_create_histogram_with_buckets( - "beacon_block_observed_slot_start_delay_time", - "Duration between the start of the block's slot and the time the block was observed.", - // [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50] - decimal_buckets(-1,2) - ); - pub static ref BEACON_BLOCK_IMPORTED_OBSERVED_DELAY_TIME: Result = try_create_histogram_with_buckets( - "beacon_block_imported_observed_delay_time", - "Duration between the time the block was observed and the time when it was imported.", - // [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5] - decimal_buckets(-2,0) - ); - pub static ref BEACON_BLOCK_HEAD_IMPORTED_DELAY_TIME: Result = try_create_histogram_with_buckets( - "beacon_block_head_imported_delay_time", - "Duration between the time the block was imported and the time when it was set as head.", - // [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5] - decimal_buckets(-2,-1) - ); - pub static ref BEACON_BLOCK_HEAD_SLOT_START_DELAY_TIME: Result = try_create_histogram_with_buckets( - "beacon_block_head_slot_start_delay_time", + pub static ref BEACON_BLOCK_DELAY_TOTAL: Result = try_create_int_gauge( + "beacon_block_delay_total", "Duration between the start of the block's slot and the time when it was set as head.", - // [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50] - decimal_buckets(-1,2) ); - pub static ref BEACON_BLOCK_HEAD_SLOT_START_DELAY_EXCEEDED_TOTAL: Result = try_create_int_counter( - "beacon_block_head_slot_start_delay_exceeded_total", - "Triggered when the duration between the start of the block's slot and the current time \ + + pub static ref BEACON_BLOCK_DELAY_OBSERVED_SLOT_START: Result = try_create_int_gauge( + "beacon_block_delay_observed_slot_start", + "Duration between the start of the block's slot and the time the block was observed.", + ); + + pub static ref BEACON_BLOB_DELAY_ALL_OBSERVED_SLOT_START: Result = try_create_int_gauge( + "beacon_blob_delay_all_observed_slot_start", + "Duration between the start of the block's slot and the time the block was observed.", + ); + + pub static ref BEACON_BLOCK_DELAY_EXECUTION_TIME: Result = try_create_int_gauge( + "beacon_block_delay_execution_time", + "The duration in verifying the block with the execution layer.", + ); + + pub static ref BEACON_BLOCK_DELAY_AVAILABLE_SLOT_START: Result = try_create_int_gauge( + "beacon_block_delay_available_slot_start", + "Duration between the time that block became available and the start of the slot.", + ); + pub static ref BEACON_BLOCK_DELAY_ATTESTABLE_SLOT_START: Result = try_create_int_gauge( + "beacon_block_delay_attestable_slot_start", + "Duration between the time that block became attestable and the start of the slot.", + ); + + pub static ref BEACON_BLOCK_DELAY_IMPORTED_TIME: Result = try_create_int_gauge( + "beacon_block_delay_imported_time", + "Duration between the time the block became available and the time when it was imported.", + ); + + pub static ref BEACON_BLOCK_DELAY_HEAD_IMPORTED_TIME: Result = try_create_int_gauge( + "beacon_block_delay_head_imported_time", + "Duration between the time that block was imported and the time when it was set as head.", + ); + pub static ref BEACON_BLOCK_DELAY_HEAD_SLOT_START_EXCEEDED_TOTAL: Result = try_create_int_counter( + "beacon_block_delay_head_slot_start_exceeded_total", + "A counter that is triggered when the duration between the start of the block's slot and the current time \ will result in failed attestations.", ); @@ -1130,11 +1148,9 @@ lazy_static! { /* * Availability related metrics */ - pub static ref BLOCK_AVAILABILITY_DELAY: Result = try_create_histogram_with_buckets( + pub static ref BLOCK_AVAILABILITY_DELAY: Result = try_create_int_gauge( "block_availability_delay", "Duration between start of the slot and the time at which all components of the block are available.", - // Create a custom bucket list for greater granularity in block delay - Ok(vec![0.1, 0.2, 0.3,0.4,0.5,0.75,1.0,1.25,1.5,1.75,2.0,2.5,3.0,3.5,4.0,5.0,6.0,7.0,8.0,9.0,10.0,15.0,20.0]) ); /* diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 22410976c9d..3e7bf7f561d 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -24,7 +24,7 @@ use payload_status::process_payload_status; pub use payload_status::PayloadStatus; use sensitive_url::SensitiveUrl; use serde::{Deserialize, Serialize}; -use slog::{crit, debug, error, info, trace, warn, Logger}; +use slog::{crit, debug, error, info, warn, Logger}; use slot_clock::SlotClock; use std::collections::HashMap; use std::fmt; @@ -1331,15 +1331,11 @@ impl ExecutionLayer { &metrics::EXECUTION_LAYER_REQUEST_TIMES, &[metrics::NEW_PAYLOAD], ); + let timer = std::time::Instant::now(); + let block_number = new_payload_request.block_number(); let block_hash = new_payload_request.block_hash(); - trace!( - self.log(), - "Issuing engine_newPayload"; - "parent_hash" => ?new_payload_request.parent_hash(), - "block_hash" => ?block_hash, - "block_number" => ?new_payload_request.block_number(), - ); + let parent_hash = new_payload_request.parent_hash(); let result = self .engine() @@ -1347,9 +1343,19 @@ impl ExecutionLayer { .await; if let Ok(status) = &result { + let status_str = <&'static str>::from(status.status); metrics::inc_counter_vec( &metrics::EXECUTION_LAYER_PAYLOAD_STATUS, - &["new_payload", status.status.into()], + &["new_payload", status_str], + ); + debug!( + self.log(), + "Processed engine_newPayload"; + "status" => status_str, + "parent_hash" => ?parent_hash, + "block_hash" => ?block_hash, + "block_number" => block_number, + "response_time_ms" => timer.elapsed().as_millis() ); } *self.inner.last_new_payload_errored.write().await = result.is_err(); diff --git a/beacon_node/network/src/metrics.rs b/beacon_node/network/src/metrics.rs index d19a41a28fc..d3804fbed8d 100644 --- a/beacon_node/network/src/metrics.rs +++ b/beacon_node/network/src/metrics.rs @@ -248,50 +248,41 @@ lazy_static! { /* * Block Delay Metrics */ - pub static ref BEACON_BLOCK_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME: Result = try_create_histogram_with_buckets( - "beacon_block_gossip_propagation_verification_delay_time", - "Duration between when the block is received and when it is verified for propagation.", - // [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5] - decimal_buckets(-3,-1) - ); - pub static ref BEACON_BLOCK_GOSSIP_SLOT_START_DELAY_TIME: Result = try_create_histogram_with_buckets( - "beacon_block_gossip_slot_start_delay_time", - "Duration between when the block is received and the start of the slot it belongs to.", - // Create a custom bucket list for greater granularity in block delay - Ok(vec![0.1, 0.2, 0.3,0.4,0.5,0.75,1.0,1.25,1.5,1.75,2.0,2.5,3.0,3.5,4.0,5.0,6.0,7.0,8.0,9.0,10.0,15.0,20.0]) - // NOTE: Previous values, which we may want to switch back to. - // [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50] - //decimal_buckets(-1,2) - + pub static ref BEACON_BLOCK_DELAY_GOSSIP: Result = try_create_int_gauge( + "beacon_block_delay_gossip", + "The first time we see this block from gossip as a delay from the start of the slot" + ); + pub static ref BEACON_BLOCK_DELAY_GOSSIP_VERIFICATION: Result = try_create_int_gauge( + "beacon_block_delay_gossip_verification", + "Keeps track of the time delay from the start of the slot to the point we propagate the block" ); - pub static ref BEACON_BLOCK_LAST_DELAY: Result = try_create_int_gauge( - "beacon_block_last_delay", - "Keeps track of the last block's delay from the start of the slot" + pub static ref BEACON_BLOCK_DELAY_FULL_VERIFICATION: Result = try_create_int_gauge( + "beacon_block_delay_full_verification", + "The time it takes to verify a beacon block." ); - pub static ref BEACON_BLOCK_GOSSIP_ARRIVED_LATE_TOTAL: Result = try_create_int_counter( - "beacon_block_gossip_arrived_late_total", + pub static ref BEACON_BLOCK_DELAY_GOSSIP_ARRIVED_LATE_TOTAL: Result = try_create_int_counter( + "beacon_block_delay_gossip_arrived_late_total", "Count of times when a gossip block arrived from the network later than the attestation deadline.", ); /* * Blob Delay Metrics */ - pub static ref BEACON_BLOB_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME: Result = try_create_histogram_with_buckets( - "beacon_blob_gossip_propagation_verification_delay_time", - "Duration between when the blob is received over gossip and when it is verified for propagation.", - // [0.001, 0.002, 0.005, 0.01, 0.02, 0.05, 0.1, 0.2, 0.5] - decimal_buckets(-3,-1) - ); - pub static ref BEACON_BLOB_GOSSIP_SLOT_START_DELAY_TIME: Result = try_create_histogram_with_buckets( - "beacon_blob_gossip_slot_start_delay_time", - "Duration between when the blob is received over gossip and the start of the slot it belongs to.", - // Create a custom bucket list for greater granularity in block delay - Ok(vec![0.1, 0.2, 0.3,0.4,0.5,0.75,1.0,1.25,1.5,1.75,2.0,2.5,3.0,3.5,4.0,5.0,6.0,7.0,8.0,9.0,10.0,15.0,20.0]) - // NOTE: Previous values, which we may want to switch back to. - // [0.1, 0.2, 0.5, 1, 2, 5, 10, 20, 50] - //decimal_buckets(-1,2) + pub static ref BEACON_BLOB_DELAY_GOSSIP: Result = try_create_int_gauge( + "beacon_blob_delay_gossip_last_delay", + "The first time we see this blob as a delay from the start of the slot" + ); + + pub static ref BEACON_BLOB_DELAY_GOSSIP_VERIFICATION: Result = try_create_int_gauge( + "beacon_blob_delay_gossip_verification", + "Keeps track of the time delay from the start of the slot to the point we propagate the blob" + ); + pub static ref BEACON_BLOB_DELAY_FULL_VERIFICATION: Result = try_create_int_gauge( + "beacon_blob_last_full_verification_delay", + "The time it takes to verify a beacon blob" ); + pub static ref BEACON_BLOB_RPC_SLOT_START_DELAY_TIME: Result = try_create_histogram_with_buckets( "beacon_blob_rpc_slot_start_delay_time", "Duration between when a blob is received over rpc and the start of the slot it belongs to.", @@ -302,10 +293,6 @@ lazy_static! { //decimal_buckets(-1,2) ); - pub static ref BEACON_BLOB_LAST_DELAY: Result = try_create_int_gauge( - "beacon_blob_last_delay", - "Keeps track of the last blob's delay from the start of the slot" - ); pub static ref BEACON_BLOB_GOSSIP_ARRIVED_LATE_TOTAL: Result = try_create_int_counter( "beacon_blob_gossip_arrived_late_total", diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index f7bba900372..7b8826bd853 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -27,7 +27,7 @@ use std::fs; use std::io::Write; use std::path::PathBuf; use std::sync::Arc; -use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use store::hot_cold_store::HotColdDBError; use tokio::sync::mpsc; use types::{ @@ -615,8 +615,7 @@ impl NetworkBeaconProcessor { let commitment = blob_sidecar.kzg_commitment; let delay = get_slot_delay_ms(seen_duration, slot, &self.chain.slot_clock); // Log metrics to track delay from other nodes on the network. - metrics::observe_duration(&metrics::BEACON_BLOB_GOSSIP_SLOT_START_DELAY_TIME, delay); - metrics::set_gauge(&metrics::BEACON_BLOB_LAST_DELAY, delay.as_millis() as i64); + metrics::set_gauge(&metrics::BEACON_BLOB_DELAY_GOSSIP, delay.as_millis() as i64); match self .chain .verify_blob_sidecar_for_gossip(blob_sidecar, blob_index) @@ -654,9 +653,9 @@ impl NetworkBeaconProcessor { .ok() .and_then(|now| now.checked_sub(seen_duration)) { - metrics::observe_duration( - &metrics::BEACON_BLOB_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME, - duration, + metrics::set_gauge( + &metrics::BEACON_BLOB_DELAY_GOSSIP_VERIFICATION, + duration.as_millis() as i64, ); } self.process_gossip_verified_blob(peer_id, gossip_verified_blob, seen_duration) @@ -747,9 +746,9 @@ impl NetworkBeaconProcessor { self: &Arc, peer_id: PeerId, verified_blob: GossipVerifiedBlob, - // This value is not used presently, but it might come in handy for debugging. _seen_duration: Duration, ) { + let processing_start_time = Instant::now(); let block_root = verified_blob.block_root(); let blob_slot = verified_blob.slot(); let blob_index = verified_blob.id().index; @@ -764,6 +763,11 @@ impl NetworkBeaconProcessor { "block_root" => %block_root ); self.chain.recompute_head_at_current_slot().await; + + metrics::set_gauge( + &metrics::BEACON_BLOB_DELAY_FULL_VERIFICATION, + processing_start_time.elapsed().as_millis() as i64, + ); } Ok(AvailabilityProcessingStatus::MissingComponents(slot, block_root)) => { trace!( @@ -865,12 +869,9 @@ impl NetworkBeaconProcessor { let block_delay = get_block_delay_ms(seen_duration, block.message(), &self.chain.slot_clock); // Log metrics to track delay from other nodes on the network. - metrics::observe_duration( - &metrics::BEACON_BLOCK_GOSSIP_SLOT_START_DELAY_TIME, - block_delay, - ); + metrics::set_gauge( - &metrics::BEACON_BLOCK_LAST_DELAY, + &metrics::BEACON_BLOCK_DELAY_GOSSIP, block_delay.as_millis() as i64, ); @@ -898,7 +899,7 @@ impl NetworkBeaconProcessor { let verified_block = match verification_result { Ok(verified_block) => { if block_delay >= self.chain.slot_clock.unagg_attestation_production_delay() { - metrics::inc_counter(&metrics::BEACON_BLOCK_GOSSIP_ARRIVED_LATE_TOTAL); + metrics::inc_counter(&metrics::BEACON_BLOCK_DELAY_GOSSIP_ARRIVED_LATE_TOTAL); debug!( self.log, "Gossip block arrived late"; @@ -923,9 +924,9 @@ impl NetworkBeaconProcessor { .ok() .and_then(|now| now.checked_sub(seen_duration)) { - metrics::observe_duration( - &metrics::BEACON_BLOCK_GOSSIP_PROPAGATION_VERIFICATION_DELAY_TIME, - duration, + metrics::set_gauge( + &metrics::BEACON_BLOCK_DELAY_GOSSIP_VERIFICATION, + duration.as_millis() as i64, ); } @@ -1130,9 +1131,9 @@ impl NetworkBeaconProcessor { verified_block: GossipVerifiedBlock, reprocess_tx: mpsc::Sender, invalid_block_storage: InvalidBlockStorage, - // This value is not used presently, but it might come in handy for debugging. _seen_duration: Duration, ) { + let processing_start_time = Instant::now(); let block = verified_block.block.block_cloned(); let block_root = verified_block.block_root; @@ -1168,6 +1169,11 @@ impl NetworkBeaconProcessor { ); self.chain.recompute_head_at_current_slot().await; + + metrics::set_gauge( + &metrics::BEACON_BLOCK_DELAY_FULL_VERIFICATION, + processing_start_time.elapsed().as_millis() as i64, + ); } Ok(AvailabilityProcessingStatus::MissingComponents(slot, block_root)) => { trace!( From 05fbbdd840304e34a34216ed8970cc32fc971f1e Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Tue, 23 Apr 2024 09:46:49 -0500 Subject: [PATCH 21/37] Electra: Add Preset, Constants, & Config (#5606) * Electra: Add Presets, Constants, & Config --- consensus/types/src/chain_spec.rs | 93 ++++++++++++++++++++++++++++++- consensus/types/src/eth_spec.rs | 81 ++++++++++++++++++++++++--- 2 files changed, 165 insertions(+), 9 deletions(-) diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index e9345ab14ea..e4f27d6873c 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -25,6 +25,7 @@ pub enum Domain { SyncCommittee, ContributionAndProof, SyncCommitteeSelectionProof, + Consolidation, ApplicationMask(ApplicationDomain), } @@ -76,6 +77,7 @@ pub struct ChainSpec { pub genesis_fork_version: [u8; 4], pub bls_withdrawal_prefix_byte: u8, pub eth1_address_withdrawal_prefix_byte: u8, + pub compounding_withdrawal_prefix_byte: u8, /* * Time parameters @@ -108,6 +110,7 @@ pub struct ChainSpec { pub(crate) domain_voluntary_exit: u32, pub(crate) domain_selection_proof: u32, pub(crate) domain_aggregate_and_proof: u32, + pub(crate) domain_consolidation: u32, /* * Fork choice @@ -177,6 +180,15 @@ pub struct ChainSpec { pub electra_fork_version: [u8; 4], /// The Electra fork epoch is optional, with `None` representing "Electra never happens". pub electra_fork_epoch: Option, + pub unset_deposit_receipts_start_index: u64, + pub full_exit_request_amount: u64, + pub min_activation_balance: u64, + pub max_effective_balance_electra: u64, + pub min_slashing_penalty_quotient_electra: u64, + pub whistleblower_reward_quotient_electra: u64, + pub max_pending_partials_per_withdrawals_sweep: u64, + pub min_per_epoch_churn_limit_electra: u64, + pub max_per_epoch_activation_exit_churn_limit: u64, /* * Networking @@ -364,7 +376,9 @@ impl ChainSpec { state: &BeaconState, ) -> u64 { let fork_name = state.fork_name_unchecked(); - if fork_name >= ForkName::Merge { + if fork_name >= ForkName::Electra { + self.min_slashing_penalty_quotient_electra + } else if fork_name >= ForkName::Merge { self.min_slashing_penalty_quotient_bellatrix } else if fork_name >= ForkName::Altair { self.min_slashing_penalty_quotient_altair @@ -418,6 +432,7 @@ impl ChainSpec { Domain::SyncCommitteeSelectionProof => self.domain_sync_committee_selection_proof, Domain::ApplicationMask(application_domain) => application_domain.get_domain_constant(), Domain::BlsToExecutionChange => self.domain_bls_to_execution_change, + Domain::Consolidation => self.domain_consolidation, } } @@ -602,6 +617,7 @@ impl ChainSpec { genesis_fork_version: [0; 4], bls_withdrawal_prefix_byte: 0x00, eth1_address_withdrawal_prefix_byte: 0x01, + compounding_withdrawal_prefix_byte: 0x02, /* * Time parameters @@ -635,6 +651,7 @@ impl ChainSpec { domain_voluntary_exit: 4, domain_selection_proof: 5, domain_aggregate_and_proof: 6, + domain_consolidation: 0x0B, /* * Fork choice @@ -709,6 +726,30 @@ impl ChainSpec { */ electra_fork_version: [0x05, 00, 00, 00], electra_fork_epoch: None, + unset_deposit_receipts_start_index: u64::MAX, + full_exit_request_amount: 0, + min_activation_balance: option_wrapper(|| { + u64::checked_pow(2, 5)?.checked_mul(u64::checked_pow(10, 9)?) + }) + .expect("calculation does not overflow"), + max_effective_balance_electra: option_wrapper(|| { + u64::checked_pow(2, 11)?.checked_mul(u64::checked_pow(10, 9)?) + }) + .expect("calculation does not overflow"), + min_slashing_penalty_quotient_electra: u64::checked_pow(2, 12) + .expect("pow does not overflow"), + whistleblower_reward_quotient_electra: u64::checked_pow(2, 12) + .expect("pow does not overflow"), + max_pending_partials_per_withdrawals_sweep: u64::checked_pow(2, 3) + .expect("pow does not overflow"), + min_per_epoch_churn_limit_electra: option_wrapper(|| { + u64::checked_pow(2, 7)?.checked_mul(u64::checked_pow(10, 9)?) + }) + .expect("calculation does not overflow"), + max_per_epoch_activation_exit_churn_limit: option_wrapper(|| { + u64::checked_pow(2, 8)?.checked_mul(u64::checked_pow(10, 9)?) + }) + .expect("calculation does not overflow"), /* * Network specific @@ -874,6 +915,7 @@ impl ChainSpec { genesis_fork_version: [0x00, 0x00, 0x00, 0x64], bls_withdrawal_prefix_byte: 0x00, eth1_address_withdrawal_prefix_byte: 0x01, + compounding_withdrawal_prefix_byte: 0x02, /* * Time parameters @@ -907,6 +949,7 @@ impl ChainSpec { domain_voluntary_exit: 4, domain_selection_proof: 5, domain_aggregate_and_proof: 6, + domain_consolidation: 0x0B, /* * Fork choice @@ -983,6 +1026,30 @@ impl ChainSpec { */ electra_fork_version: [0x05, 0x00, 0x00, 0x64], electra_fork_epoch: None, + unset_deposit_receipts_start_index: u64::MAX, + full_exit_request_amount: 0, + min_activation_balance: option_wrapper(|| { + u64::checked_pow(2, 5)?.checked_mul(u64::checked_pow(10, 9)?) + }) + .expect("calculation does not overflow"), + max_effective_balance_electra: option_wrapper(|| { + u64::checked_pow(2, 11)?.checked_mul(u64::checked_pow(10, 9)?) + }) + .expect("calculation does not overflow"), + min_slashing_penalty_quotient_electra: u64::checked_pow(2, 12) + .expect("pow does not overflow"), + whistleblower_reward_quotient_electra: u64::checked_pow(2, 12) + .expect("pow does not overflow"), + max_pending_partials_per_withdrawals_sweep: u64::checked_pow(2, 3) + .expect("pow does not overflow"), + min_per_epoch_churn_limit_electra: option_wrapper(|| { + u64::checked_pow(2, 7)?.checked_mul(u64::checked_pow(10, 9)?) + }) + .expect("calculation does not overflow"), + max_per_epoch_activation_exit_churn_limit: option_wrapper(|| { + u64::checked_pow(2, 8)?.checked_mul(u64::checked_pow(10, 9)?) + }) + .expect("calculation does not overflow"), /* * Network specific @@ -1206,6 +1273,13 @@ pub struct Config { #[serde(default = "default_blob_sidecar_subnet_count")] #[serde(with = "serde_utils::quoted_u64")] blob_sidecar_subnet_count: u64, + + #[serde(default = "default_min_per_epoch_churn_limit_electra")] + #[serde(with = "serde_utils::quoted_u64")] + min_per_epoch_churn_limit_electra: u64, + #[serde(default = "default_max_per_epoch_activation_exit_churn_limit")] + #[serde(with = "serde_utils::quoted_u64")] + max_per_epoch_activation_exit_churn_limit: u64, } fn default_bellatrix_fork_version() -> [u8; 4] { @@ -1320,6 +1394,14 @@ const fn default_blob_sidecar_subnet_count() -> u64 { 6 } +const fn default_min_per_epoch_churn_limit_electra() -> u64 { + 128_000_000_000 +} + +const fn default_max_per_epoch_activation_exit_churn_limit() -> u64 { + 256_000_000_000 +} + const fn default_epochs_per_subnet_subscription() -> u64 { 256 } @@ -1496,6 +1578,10 @@ impl Config { max_request_blob_sidecars: spec.max_request_blob_sidecars, min_epochs_for_blob_sidecars_requests: spec.min_epochs_for_blob_sidecars_requests, blob_sidecar_subnet_count: spec.blob_sidecar_subnet_count, + + min_per_epoch_churn_limit_electra: spec.min_per_epoch_churn_limit_electra, + max_per_epoch_activation_exit_churn_limit: spec + .max_per_epoch_activation_exit_churn_limit, } } @@ -1563,6 +1649,8 @@ impl Config { max_request_blob_sidecars, min_epochs_for_blob_sidecars_requests, blob_sidecar_subnet_count, + min_per_epoch_churn_limit_electra, + max_per_epoch_activation_exit_churn_limit, } = self; if preset_base != E::spec_name().to_string().as_str() { @@ -1623,6 +1711,8 @@ impl Config { max_request_blob_sidecars, min_epochs_for_blob_sidecars_requests, blob_sidecar_subnet_count, + min_per_epoch_churn_limit_electra, + max_per_epoch_activation_exit_churn_limit, // We need to re-derive any values that might have changed in the config. max_blocks_by_root_request: max_blocks_by_root_request_common(max_request_blocks), @@ -1695,6 +1785,7 @@ mod tests { &spec, ); test_domain(Domain::SyncCommittee, spec.domain_sync_committee, &spec); + test_domain(Domain::Consolidation, spec.domain_consolidation, &spec); // The builder domain index is zero let builder_domain_pre_mask = [0; 4]; diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index a2972b722a2..a700c5e9abb 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -3,8 +3,9 @@ use crate::*; use safe_arith::SafeArith; use serde::{Deserialize, Serialize}; use ssz_types::typenum::{ - bit::B0, UInt, U0, U1024, U1048576, U1073741824, U1099511627776, U128, U131072, U16, U16777216, - U2, U2048, U256, U32, U4, U4096, U512, U6, U625, U64, U65536, U8, U8192, + bit::B0, UInt, U0, U1, U1024, U1048576, U1073741824, U1099511627776, U128, U131072, U134217728, + U16, U16777216, U2, U2048, U256, U262144, U32, U4, U4096, U512, U6, U625, U64, U65536, U8, + U8192, }; use ssz_types::typenum::{U17, U9}; use std::fmt::{self, Debug}; @@ -137,7 +138,14 @@ pub trait EthSpec: /* * New in Electra */ - type ElectraPlaceholder: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type PendingBalanceDepositsLimit: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type PendingPartialWithdrawalsLimit: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type PendingConsolidationsLimit: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type MaxConsolidations: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type MaxDepositReceiptsPerPayload: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type MaxAttesterSlashingsElectra: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type MaxAttestationsElectra: Unsigned + Clone + Sync + Send + Debug + PartialEq; + type MaxWithdrawalRequestsPerPayload: Unsigned + Clone + Sync + Send + Debug + PartialEq; fn default_spec() -> ChainSpec; @@ -284,8 +292,44 @@ pub trait EthSpec: Self::KzgCommitmentInclusionProofDepth::to_usize() } - fn electra_placeholder() -> usize { - Self::ElectraPlaceholder::to_usize() + /// Returns the `PENDING_BALANCE_DEPOSITS_LIMIT` constant for this specification. + fn pending_balance_deposits_limit() -> usize { + Self::PendingBalanceDepositsLimit::to_usize() + } + + /// Returns the `PENDING_PARTIAL_WITHDRAWALS_LIMIT` constant for this specification. + fn pending_partial_withdrawals_limit() -> usize { + Self::PendingPartialWithdrawalsLimit::to_usize() + } + + /// Returns the `PENDING_CONSOLIDATIONS_LIMIT` constant for this specification. + fn pending_consolidations_limit() -> usize { + Self::PendingConsolidationsLimit::to_usize() + } + + /// Returns the `MAX_CONSOLIDATIONS` constant for this specification. + fn max_consolidations() -> usize { + Self::MaxConsolidations::to_usize() + } + + /// Returns the `MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD` constant for this specification. + fn max_deposit_receipts_per_payload() -> usize { + Self::MaxDepositReceiptsPerPayload::to_usize() + } + + /// Returns the `MAX_ATTESTER_SLASHINGS_ELECTRA` constant for this specification. + fn max_attester_slashings_electra() -> usize { + Self::MaxAttesterSlashingsElectra::to_usize() + } + + /// Returns the `MAX_ATTESTATIONS_ELECTRA` constant for this specification. + fn max_attestations_electra() -> usize { + Self::MaxAttestationsElectra::to_usize() + } + + /// Returns the `MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD` constant for this specification. + fn max_withdrawal_requests_per_payload() -> usize { + Self::MaxWithdrawalRequestsPerPayload::to_usize() } } @@ -337,7 +381,14 @@ impl EthSpec for MainnetEthSpec { type SlotsPerEth1VotingPeriod = U2048; // 64 epochs * 32 slots per epoch type MaxBlsToExecutionChanges = U16; type MaxWithdrawalsPerPayload = U16; - type ElectraPlaceholder = U16; + type PendingBalanceDepositsLimit = U134217728; + type PendingPartialWithdrawalsLimit = U134217728; + type PendingConsolidationsLimit = U262144; + type MaxConsolidations = U1; + type MaxDepositReceiptsPerPayload = U8192; + type MaxAttesterSlashingsElectra = U1; + type MaxAttestationsElectra = U8; + type MaxWithdrawalRequestsPerPayload = U16; fn default_spec() -> ChainSpec { ChainSpec::mainnet() @@ -390,7 +441,14 @@ impl EthSpec for MinimalEthSpec { MaxBlsToExecutionChanges, MaxBlobsPerBlock, BytesPerFieldElement, - ElectraPlaceholder + PendingBalanceDepositsLimit, + PendingPartialWithdrawalsLimit, + PendingConsolidationsLimit, + MaxConsolidations, + MaxDepositReceiptsPerPayload, + MaxAttesterSlashingsElectra, + MaxAttestationsElectra, + MaxWithdrawalRequestsPerPayload }); fn default_spec() -> ChainSpec { @@ -442,7 +500,14 @@ impl EthSpec for GnosisEthSpec { type BytesPerFieldElement = U32; type BytesPerBlob = U131072; type KzgCommitmentInclusionProofDepth = U17; - type ElectraPlaceholder = U16; + type PendingBalanceDepositsLimit = U134217728; + type PendingPartialWithdrawalsLimit = U134217728; + type PendingConsolidationsLimit = U262144; + type MaxConsolidations = U1; + type MaxDepositReceiptsPerPayload = U8192; + type MaxAttesterSlashingsElectra = U1; + type MaxAttestationsElectra = U8; + type MaxWithdrawalRequestsPerPayload = U16; fn default_spec() -> ChainSpec { ChainSpec::gnosis() From 76460ba838cf806613cb788d489020ae71c11ea0 Mon Sep 17 00:00:00 2001 From: antondlr Date: Tue, 23 Apr 2024 19:58:47 +0200 Subject: [PATCH 22/37] Only `portable` builds (docker) (#5614) * portable builds by default, build multiarch lcli --- .github/workflows/docker.yml | 113 ++++++++++++++++++----------------- lcli/Dockerfile.cross | 6 ++ 2 files changed, 65 insertions(+), 54 deletions(-) create mode 100644 lcli/Dockerfile.cross diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 54b355e631d..d1a8c9f6144 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -15,8 +15,6 @@ concurrency: env: DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }} DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} - IMAGE_NAME: ${{ github.repository_owner}}/lighthouse - LCLI_IMAGE_NAME: ${{ github.repository_owner }}/lcli # Enable self-hosted runners for the sigp repo only. SELF_HOSTED_RUNNERS: ${{ github.repository == 'sigp/lighthouse' }} @@ -49,19 +47,15 @@ jobs: VERSION: ${{ env.VERSION }} VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }} build-docker-single-arch: - name: build-docker-${{ matrix.binary }}${{ matrix.features.version_suffix }} + name: build-docker-${{ matrix.binary }}-${{ matrix.cpu_arch }}${{ matrix.features.version_suffix }} # Use self-hosted runners only on the sigp repo. runs-on: ${{ github.repository == 'sigp/lighthouse' && fromJson('["self-hosted", "linux", "release"]') || 'ubuntu-22.04' }} strategy: matrix: - binary: [aarch64, - aarch64-portable, - x86_64, - x86_64-portable] - features: [ - {version_suffix: "", env: "gnosis,slasher-lmdb,slasher-mdbx,jemalloc"}, - {version_suffix: "-dev", env: "jemalloc,spec-minimal"} - ] + binary: [lighthouse, + lcli] + cpu_arch: [aarch64, + x86_64] include: - profile: maxperf @@ -69,7 +63,6 @@ jobs: env: VERSION: ${{ needs.extract-version.outputs.VERSION }} VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} - FEATURE_SUFFIX: ${{ matrix.features.version_suffix }} steps: - uses: actions/checkout@v4 - name: Update Rust @@ -78,27 +71,40 @@ jobs: - name: Dockerhub login run: | echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - - name: Cross build Lighthouse binary + + - name: Sets env vars for Lighthouse + if: startsWith(matrix.binary, 'lighthouse') + run: | + echo "CROSS_FEATURES=gnosis,spec-minimal,slasher-lmdb,jemalloc" >> $GITHUB_ENV + + - name: Set `make` command for lighthouse + if: startsWith(matrix.binary, 'lighthouse') + run: | + echo "MAKE_CMD=build-${{ matrix.cpu_arch }}-portable" >> $GITHUB_ENV + + - name: Set `make` command for lcli + if: startsWith(matrix.binary, 'lcli') + run: | + echo "MAKE_CMD=build-lcli-${{ matrix.cpu_arch }}" >> $GITHUB_ENV + + - name: Cross build binaries run: | cargo install cross - env CROSS_PROFILE=${{ matrix.profile }} CROSS_FEATURES=${{ matrix.features.env }} make build-${{ matrix.binary }} + env CROSS_PROFILE=${{ matrix.profile }} CROSS_FEATURES=${{ env.CROSS_FEATURES }} make ${{ env.MAKE_CMD }} + - name: Make bin dir run: mkdir ./bin - - name: Move cross-built binary into Docker scope (if ARM) - if: startsWith(matrix.binary, 'aarch64') - run: mv ./target/aarch64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin - - name: Move cross-built binary into Docker scope (if x86_64) - if: startsWith(matrix.binary, 'x86_64') - run: mv ./target/x86_64-unknown-linux-gnu/${{ matrix.profile }}/lighthouse ./bin + + - name: Move cross-built binary into Docker scope + run: mv ./target/${{ matrix.cpu_arch }}-unknown-linux-gnu/${{ matrix.profile }}/${{ matrix.binary }} ./bin + - name: Map aarch64 to arm64 short arch - if: startsWith(matrix.binary, 'aarch64') + if: startsWith(matrix.cpu_arch, 'aarch64') run: echo "SHORT_ARCH=arm64" >> $GITHUB_ENV + - name: Map x86_64 to amd64 short arch - if: startsWith(matrix.binary, 'x86_64') + if: startsWith(matrix.cpu_arch, 'x86_64') run: echo "SHORT_ARCH=amd64" >> $GITHUB_ENV; - - name: Set modernity suffix - if: endsWith(matrix.binary, '-portable') != true - run: echo "MODERNITY_SUFFIX=-modern" >> $GITHUB_ENV; - name: Install QEMU if: env.SELF_HOSTED_RUNNERS == 'false' @@ -108,22 +114,41 @@ jobs: if: env.SELF_HOSTED_RUNNERS == 'false' uses: docker/setup-buildx-action@v3 - - name: Build and push + - name: Build and push (Lighthouse) + if: startsWith(matrix.binary, 'lighthouse') uses: docker/build-push-action@v5 with: file: ./Dockerfile.cross context: . platforms: linux/${{ env.SHORT_ARCH }} push: true - tags: ${{ env.IMAGE_NAME }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}${{ env.MODERNITY_SUFFIX }}${{ env.FEATURE_SUFFIX }} + tags: | + ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }} + ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}-dev + ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}-modern + ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }}-modern-dev + + - name: Build and push (lcli) + if: startsWith(matrix.binary, 'lcli') + uses: docker/build-push-action@v5 + with: + file: ./lcli/Dockerfile.cross + context: . + platforms: linux/${{ env.SHORT_ARCH }} + push: true + + tags: | + ${{ github.repository_owner}}/${{ matrix.binary }}:${{ env.VERSION }}-${{ env.SHORT_ARCH }}${{ env.VERSION_SUFFIX }} + build-docker-multiarch: - name: build-docker-multiarch${{ matrix.modernity }} + name: build-docker-${{ matrix.binary }}-multiarch runs-on: ubuntu-22.04 - needs: [build-docker-single-arch, extract-version] strategy: matrix: - modernity: ["", "-modern"] + binary: [lighthouse, + lcli] + needs: [build-docker-single-arch, extract-version] env: VERSION: ${{ needs.extract-version.outputs.VERSION }} VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} @@ -135,29 +160,9 @@ jobs: run: | echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - - name: Create and push multiarch manifest + - name: Create and push multiarch manifests run: | - docker buildx imagetools create -t ${IMAGE_NAME}:${VERSION}${VERSION_SUFFIX}${{ matrix.modernity }} \ - ${IMAGE_NAME}:${VERSION}-arm64${VERSION_SUFFIX}${{ matrix.modernity }} \ - ${IMAGE_NAME}:${VERSION}-amd64${VERSION_SUFFIX}${{ matrix.modernity }}; + docker buildx imagetools create -t ${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}${VERSION_SUFFIX} \ + ${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}-arm64${VERSION_SUFFIX} \ + ${{ github.repository_owner}}/${{ matrix.binary }}:${VERSION}-amd64${VERSION_SUFFIX}; - build-docker-lcli: - runs-on: ubuntu-22.04 - needs: [extract-version] - env: - VERSION: ${{ needs.extract-version.outputs.VERSION }} - VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }} - steps: - - uses: actions/checkout@v4 - - name: Dockerhub login - run: | - echo "${DOCKER_PASSWORD}" | docker login --username ${DOCKER_USERNAME} --password-stdin - - name: Build lcli and push - uses: docker/build-push-action@v5 - with: - build-args: | - FEATURES=portable - context: . - push: true - file: ./lcli/Dockerfile - tags: ${{ env.LCLI_IMAGE_NAME }}:${{ env.VERSION }}${{ env.VERSION_SUFFIX }} diff --git a/lcli/Dockerfile.cross b/lcli/Dockerfile.cross new file mode 100644 index 00000000000..979688c9cf6 --- /dev/null +++ b/lcli/Dockerfile.cross @@ -0,0 +1,6 @@ +# This image is meant to enable cross-architecture builds. +# It assumes the lcli binary has already been +# compiled for `$TARGETPLATFORM` and moved to `./bin`. +FROM --platform=$TARGETPLATFORM ubuntu:22.04 +RUN apt update && apt -y upgrade && apt clean && rm -rf /var/lib/apt/lists/* +COPY ./bin/lcli /usr/local/bin/lcli From 4cad1fcbbe1315e2d7d6171a797db7ce4e806f72 Mon Sep 17 00:00:00 2001 From: antondlr Date: Tue, 23 Apr 2024 22:12:55 +0200 Subject: [PATCH 23/37] Add missing lcli targets in makefile (#5633) * Add missing lcli targets to Makefile --- Makefile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Makefile b/Makefile index 4b2d0f6c5d5..4072ab1e6d8 100644 --- a/Makefile +++ b/Makefile @@ -82,6 +82,11 @@ build-aarch64: build-aarch64-portable: cross build --bin lighthouse --target aarch64-unknown-linux-gnu --features "portable,$(CROSS_FEATURES)" --profile "$(CROSS_PROFILE)" --locked +build-lcli-x86_64: + cross build --bin lcli --target x86_64-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked +build-lcli-aarch64: + cross build --bin lcli --target aarch64-unknown-linux-gnu --features "portable" --profile "$(CROSS_PROFILE)" --locked + # Create a `.tar.gz` containing a binary for a specific target. define tarball_release_binary cp $(1)/lighthouse $(BIN_DIR)/lighthouse From 61962898e207e03bb2826f44fd994986c9bfaa20 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 24 Apr 2024 11:22:36 +1000 Subject: [PATCH 24/37] In-memory tree states (#5533) * Consensus changes * EF tests * lcli * common and watch * account manager * cargo * fork choice * promise cache * beacon chain * interop genesis * http api * lighthouse * op pool * beacon chain misc * parallel state cache * store * fix issues in store * IT COMPILES * Remove some unnecessary module qualification * Revert Arced pubkey optimization (#5536) * Merge remote-tracking branch 'origin/unstable' into tree-states-memory * Fix caching, rebasing and some tests * Remove unused deps * Merge remote-tracking branch 'origin/unstable' into tree-states-memory * Small cleanups * Revert shuffling cache/promise cache changes * Fix state advance bugs * Fix shuffling tests * Remove some resolved FIXMEs * Remove StateProcessingStrategy * Optimise withdrawals calculation * Don't reorg if state cache is missed * Remove inconsistent state func * Fix beta compiler * Rebase early, rebase often * Fix state caching behaviour * Update to milhouse release * Fix on-disk consensus context format * Merge remote-tracking branch 'origin/unstable' into tree-states-memory * Squashed commit of the following: commit 3a16649023e2d6d40d704dbe958cec712544aba9 Author: Michael Sproul Date: Thu Apr 18 14:26:09 2024 +1000 Fix on-disk consensus context format * Keep indexed attestations, thanks Sean * Merge branch 'on-disk-consensus-context' into tree-states-memory * Merge branch 'unstable' into tree-states-memory * Address half of Sean's review * More simplifications from Sean's review * Cache state after get_advanced_hot_state --- Cargo.lock | 86 +- Cargo.toml | 12 +- beacon_node/beacon_chain/Cargo.toml | 78 +- beacon_node/beacon_chain/src/beacon_chain.rs | 325 +++----- .../beacon_chain/src/beacon_proposer_cache.rs | 8 +- .../beacon_chain/src/beacon_snapshot.rs | 25 +- .../beacon_chain/src/blob_verification.rs | 125 +-- .../beacon_chain/src/block_verification.rs | 152 ++-- beacon_node/beacon_chain/src/builder.rs | 19 +- .../beacon_chain/src/canonical_head.rs | 114 +-- beacon_node/beacon_chain/src/chain_config.rs | 3 - .../overflow_lru_cache.rs | 2 +- .../state_lru_cache.rs | 3 +- beacon_node/beacon_chain/src/errors.rs | 4 + beacon_node/beacon_chain/src/eth1_chain.rs | 29 +- beacon_node/beacon_chain/src/fork_revert.rs | 3 +- beacon_node/beacon_chain/src/lib.rs | 1 - beacon_node/beacon_chain/src/metrics.rs | 17 +- .../beacon_chain/src/shuffling_cache.rs | 2 +- .../beacon_chain/src/state_advance_timer.rs | 185 ++--- .../src/sync_committee_rewards.rs | 42 +- beacon_node/beacon_chain/src/test_utils.rs | 17 +- .../src/validator_pubkey_cache.rs | 5 +- .../beacon_chain/tests/block_verification.rs | 5 +- .../tests/payload_invalidation.rs | 4 +- beacon_node/beacon_chain/tests/rewards.rs | 28 +- beacon_node/beacon_chain/tests/store_tests.rs | 72 +- beacon_node/genesis/src/interop.rs | 10 +- beacon_node/http_api/src/attester_duties.rs | 7 +- .../http_api/src/block_packing_efficiency.rs | 2 +- beacon_node/http_api/src/lib.rs | 14 +- beacon_node/http_api/src/proposer_duties.rs | 5 +- beacon_node/http_api/src/state_id.rs | 5 +- beacon_node/http_api/tests/fork_tests.rs | 5 +- beacon_node/http_api/tests/tests.rs | 24 +- beacon_node/operation_pool/src/lib.rs | 8 +- beacon_node/src/config.rs | 7 +- beacon_node/store/Cargo.toml | 4 + beacon_node/store/src/chunked_vector.rs | 68 +- beacon_node/store/src/config.rs | 4 + beacon_node/store/src/errors.rs | 16 +- beacon_node/store/src/hot_cold_store.rs | 333 +++++--- beacon_node/store/src/impls/beacon_state.rs | 4 +- beacon_node/store/src/iter.rs | 10 +- beacon_node/store/src/lib.rs | 1 + beacon_node/store/src/partial_beacon_state.rs | 33 +- beacon_node/store/src/reconstruct.rs | 3 +- beacon_node/store/src/state_cache.rs | 303 +++++++ common/task_executor/Cargo.toml | 1 + common/task_executor/src/test_utils.rs | 3 +- consensus/state_processing/src/all_caches.rs | 6 +- .../state_processing/src/block_replayer.rs | 100 +-- .../src/common/initiate_validator_exit.rs | 3 +- consensus/state_processing/src/genesis.rs | 6 +- consensus/state_processing/src/lib.rs | 2 +- .../src/per_block_processing.rs | 6 +- .../src/per_block_processing/errors.rs | 7 + .../process_operations.rs | 7 +- .../src/per_block_processing/tests.rs | 7 +- .../altair/participation_flag_updates.rs | 9 +- .../capella/historical_summaries_update.rs | 3 + .../effective_balance_updates.rs | 6 +- .../epoch_processing_summary.rs | 16 +- .../src/per_epoch_processing/errors.rs | 9 +- .../historical_roots_update.rs | 2 +- .../src/per_epoch_processing/resets.rs | 4 +- .../src/per_epoch_processing/single_pass.rs | 66 +- .../state_processing/src/upgrade/altair.rs | 9 +- .../state_processing/src/upgrade/capella.rs | 5 +- .../state_processing/src/upgrade/deneb.rs | 1 - .../state_processing/src/upgrade/electra.rs | 1 - .../state_processing/src/upgrade/merge.rs | 1 - consensus/types/Cargo.toml | 4 +- consensus/types/benches/benches.rs | 46 +- consensus/types/examples/clone_state.rs | 51 -- consensus/types/examples/ssz_encode_state.rs | 54 -- consensus/types/examples/tree_hash_state.rs | 56 -- consensus/types/src/beacon_state.rs | 777 ++++++++++++------ .../types/src/beacon_state/clone_config.rs | 47 -- .../types/src/beacon_state/committee_cache.rs | 7 +- .../src/beacon_state/committee_cache/tests.rs | 2 +- .../types/src/beacon_state/exit_cache.rs | 11 +- consensus/types/src/beacon_state/iter.rs | 4 +- .../types/src/beacon_state/pubkey_cache.rs | 16 +- .../types/src/beacon_state/slashings_cache.rs | 6 +- consensus/types/src/beacon_state/tests.rs | 240 +----- .../types/src/beacon_state/tree_hash_cache.rs | 645 --------------- consensus/types/src/blob_sidecar.rs | 3 +- .../types/src/execution_payload_header.rs | 24 +- consensus/types/src/historical_batch.rs | 6 +- consensus/types/src/lib.rs | 6 +- consensus/types/src/light_client_bootstrap.rs | 7 +- .../types/src/light_client_finality_update.rs | 1 + consensus/types/src/light_client_update.rs | 7 + consensus/types/src/test_utils/test_random.rs | 2 +- consensus/types/src/tree_hash_impls.rs | 165 ---- lcli/src/new_testnet.rs | 2 +- lcli/src/replace_state_pubkeys.rs | 7 +- lcli/src/skip_slots.rs | 4 +- lcli/src/transition_blocks.rs | 7 +- lighthouse/tests/beacon_node.rs | 20 - testing/ef_tests/src/case_result.rs | 3 + .../src/cases/merkle_proof_validity.rs | 5 +- testing/ef_tests/src/cases/sanity_blocks.rs | 4 +- testing/ef_tests/src/cases/ssz_generic.rs | 28 +- testing/ef_tests/src/cases/ssz_static.rs | 1 - testing/ef_tests/src/cases/transition.rs | 3 +- testing/state_transition_vectors/src/exit.rs | 5 +- 108 files changed, 2032 insertions(+), 2756 deletions(-) create mode 100644 beacon_node/store/src/state_cache.rs delete mode 100644 consensus/types/examples/clone_state.rs delete mode 100644 consensus/types/examples/ssz_encode_state.rs delete mode 100644 consensus/types/examples/tree_hash_state.rs delete mode 100644 consensus/types/src/beacon_state/clone_config.rs delete mode 100644 consensus/types/src/beacon_state/tree_hash_cache.rs delete mode 100644 consensus/types/src/tree_hash_impls.rs diff --git a/Cargo.lock b/Cargo.lock index 4e3557c7efb..d63773d9a94 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -361,6 +361,15 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457" +[[package]] +name = "archery" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a8da9bc4c4053ee067669762bcaeea6e241841295a2b6c948312dad6ef4cc02" +dependencies = [ + "static_assertions", +] + [[package]] name = "ark-ff" version = "0.3.0" @@ -2810,13 +2819,13 @@ dependencies = [ [[package]] name = "ethereum_hashing" -version = "1.0.0-beta.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233dc6f434ce680dbabf4451ee3380cec46cb3c45d66660445a435619710dd35" +checksum = "6ea7b408432c13f71af01197b1d3d0069c48a27bfcfbe72a81fc346e47f6defb" dependencies = [ "cpufeatures", "lazy_static", - "ring 0.16.20", + "ring 0.17.8", "sha2 0.10.8", ] @@ -5456,6 +5465,29 @@ dependencies = [ "quote", ] +[[package]] +name = "milhouse" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3826d3602a3674b07e080ce1982350e454ec253d73f156bd927ac1b652293f4d" +dependencies = [ + "arbitrary", + "derivative", + "ethereum-types 0.14.1", + "ethereum_hashing", + "ethereum_ssz", + "ethereum_ssz_derive", + "itertools", + "parking_lot 0.12.1", + "rayon", + "serde", + "smallvec", + "tree_hash", + "triomphe", + "typenum", + "vec_map", +] + [[package]] name = "mime" version = "0.3.17" @@ -7109,6 +7141,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "rpds" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ef5140bcb576bfd6d56cd2de709a7d17851ac1f3805e67fe9d99e42a11821f" +dependencies = [ + "archery", +] + [[package]] name = "rtnetlink" version = "0.10.1" @@ -8123,9 +8164,9 @@ dependencies = [ [[package]] name = "ssz_types" -version = "0.5.4" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "382939886cb24ee8ac885d09116a60f6262d827c7a9e36012b4f6d3d0116d0b3" +checksum = "625b20de2d4b3891e6972f4ce5061cb11bd52b3479270c4b177c134b571194a9" dependencies = [ "arbitrary", "derivative", @@ -8139,6 +8180,12 @@ dependencies = [ "typenum", ] +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "state_processing" version = "0.2.0" @@ -8189,6 +8236,7 @@ name = "store" version = "0.2.0" dependencies = [ "beacon_chain", + "bls", "db-key", "directory", "ethereum_ssz", @@ -8197,11 +8245,14 @@ dependencies = [ "lazy_static", "leveldb", "lighthouse_metrics", + "logging", "lru", "parking_lot 0.12.1", + "safe_arith", "serde", "slog", "sloggers", + "smallvec", "state_processing", "strum", "tempfile", @@ -8267,9 +8318,9 @@ checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" [[package]] name = "superstruct" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75b9e5728aa1a87141cefd4e7509903fc01fa0dcb108022b1e841a67c5159fc5" +checksum = "6f4e1f478a7728f8855d7e620e9a152cf8932c6614f86564c886f9b8141f3201" dependencies = [ "darling 0.13.4", "itertools", @@ -8415,6 +8466,7 @@ dependencies = [ "futures", "lazy_static", "lighthouse_metrics", + "logging", "slog", "sloggers", "tokio", @@ -8955,9 +9007,9 @@ dependencies = [ [[package]] name = "tree_hash" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c998ac5fe2b07c025444bdd522e6258110b63861c6698eedc610c071980238d" +checksum = "134d6b24a5b829f30b5ee7de05ba7384557f5f6b00e29409cdf2392f93201bfa" dependencies = [ "ethereum-types 0.14.1", "ethereum_hashing", @@ -8966,9 +9018,9 @@ dependencies = [ [[package]] name = "tree_hash_derive" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84303a9c7cda5f085a3ed9cd241d1e95e04d88aab1d679b02f212e653537ba86" +checksum = "9ce7bccc538359a213436af7bc95804bdbf1c2a21d80e22953cbe9e096837ff1" dependencies = [ "darling 0.13.4", "quote", @@ -8985,6 +9037,16 @@ dependencies = [ "rlp", ] +[[package]] +name = "triomphe" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee8098afad3fb0c54a9007aab6804558410503ad676d4633f9c2559a00ac0f" +dependencies = [ + "serde", + "stable_deref_trait", +] + [[package]] name = "try-lock" version = "0.2.5" @@ -9024,12 +9086,14 @@ dependencies = [ "maplit", "merkle_proof", "metastruct", + "milhouse", "parking_lot 0.12.1", "paste", "rand", "rand_xorshift", "rayon", "regex", + "rpds", "rusqlite", "safe_arith", "serde", diff --git a/Cargo.toml b/Cargo.toml index 69f6835f575..296dadc1f66 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -114,7 +114,7 @@ discv5 = { version = "0.4.1", features = ["libp2p"] } env_logger = "0.9" error-chain = "0.12" ethereum-types = "0.14" -ethereum_hashing = "1.0.0-beta.2" +ethereum_hashing = "0.6.0" ethereum_serde_utils = "0.5.2" ethereum_ssz = "0.5" ethereum_ssz_derive = "0.5" @@ -132,6 +132,7 @@ libsecp256k1 = "0.7" log = "0.4" lru = "0.12" maplit = "1" +milhouse = "0.1" num_cpus = "1" parking_lot = "0.12" paste = "1" @@ -144,6 +145,7 @@ rayon = "1.7" regex = "1" reqwest = { version = "0.11", default-features = false, features = ["blocking", "json", "stream", "rustls-tls", "native-tls-vendored"] } ring = "0.16" +rpds = "0.11" rusqlite = { version = "0.28", features = ["bundled"] } serde = { version = "1", features = ["derive"] } serde_json = "1" @@ -156,9 +158,9 @@ slog-term = "2" sloggers = { version = "2", features = ["json"] } smallvec = "1.11.2" snap = "1" -ssz_types = "0.5" +ssz_types = "0.6" strum = { version = "0.24", features = ["derive"] } -superstruct = "0.6" +superstruct = "0.7" syn = "1" sysinfo = "0.26" tempfile = "3" @@ -170,8 +172,8 @@ tracing-appender = "0.2" tracing-core = "0.1" tracing-log = "0.2" tracing-subscriber = { version = "0.3", features = ["env-filter"] } -tree_hash = "0.5" -tree_hash_derive = "0.5" +tree_hash = "0.6" +tree_hash_derive = "0.6" url = "2" uuid = { version = "0.8", features = ["serde", "v4"] } warp = { version = "0.3.7", default-features = false, features = ["tls"] } diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 43c2c896f71..9c7c7febc5f 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -19,58 +19,58 @@ environment = { workspace = true } serde_json = { workspace = true } [dependencies] -serde_json = { workspace = true } +bitvec = { workspace = true } +bls = { workspace = true } +derivative = { workspace = true } +eth1 = { workspace = true } +eth2 = { workspace = true } eth2_network_config = { workspace = true } -merkle_proof = { workspace = true } -store = { workspace = true } -parking_lot = { workspace = true } +ethereum_hashing = { workspace = true } +ethereum_serde_utils = { workspace = true } +ethereum_ssz = { workspace = true } +ethereum_ssz_derive = { workspace = true } +execution_layer = { workspace = true } +fork_choice = { workspace = true } +futures = { workspace = true } +genesis = { workspace = true } +hex = { workspace = true } +int_to_bytes = { workspace = true } +itertools = { workspace = true } +kzg = { workspace = true } lazy_static = { workspace = true } -smallvec = { workspace = true } lighthouse_metrics = { workspace = true } +logging = { workspace = true } +lru = { workspace = true } +merkle_proof = { workspace = true } +oneshot_broadcast = { path = "../../common/oneshot_broadcast/" } operation_pool = { workspace = true } +parking_lot = { workspace = true } +proto_array = { workspace = true } +rand = { workspace = true } rayon = { workspace = true } +safe_arith = { workspace = true } +sensitive_url = { workspace = true } serde = { workspace = true } -ethereum_serde_utils = { workspace = true } +serde_json = { workspace = true } +slasher = { workspace = true } slog = { workspace = true } +slog-async = { workspace = true } +slog-term = { workspace = true } sloggers = { workspace = true } slot_clock = { workspace = true } -ethereum_hashing = { workspace = true } -ethereum_ssz = { workspace = true } +smallvec = { workspace = true } ssz_types = { workspace = true } -ethereum_ssz_derive = { workspace = true } state_processing = { workspace = true } -tree_hash_derive = { workspace = true } -tree_hash = { workspace = true } -types = { workspace = true } -tokio = { workspace = true } -tokio-stream = { workspace = true } -eth1 = { workspace = true } -futures = { workspace = true } -genesis = { workspace = true } -int_to_bytes = { workspace = true } -rand = { workspace = true } -proto_array = { workspace = true } -lru = { workspace = true } -tempfile = { workspace = true } -bitvec = { workspace = true } -bls = { workspace = true } -kzg = { workspace = true } -safe_arith = { workspace = true } -fork_choice = { workspace = true } -task_executor = { workspace = true } -derivative = { workspace = true } -itertools = { workspace = true } -slasher = { workspace = true } -eth2 = { workspace = true } +store = { workspace = true } strum = { workspace = true } -logging = { workspace = true } -execution_layer = { workspace = true } -sensitive_url = { workspace = true } superstruct = { workspace = true } -hex = { workspace = true } -oneshot_broadcast = { path = "../../common/oneshot_broadcast/" } -slog-term = { workspace = true } -slog-async = { workspace = true } +task_executor = { workspace = true } +tempfile = { workspace = true } +tokio = { workspace = true } +tokio-stream = { workspace = true } +tree_hash = { workspace = true } +tree_hash_derive = { workspace = true } +types = { workspace = true } [[test]] name = "beacon_chain_tests" diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index c59c5e8ed10..c65dfdb449f 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -58,7 +58,6 @@ use crate::persisted_beacon_chain::{PersistedBeaconChain, DUMMY_CANONICAL_HEAD_B use crate::persisted_fork_choice::PersistedForkChoice; use crate::pre_finalization_cache::PreFinalizationBlockCache; use crate::shuffling_cache::{BlockShufflingIds, ShufflingCache}; -use crate::snapshot_cache::{BlockProductionPreState, SnapshotCache}; use crate::sync_committee_verification::{ Error as SyncCommitteeError, VerifiedSyncCommitteeMessage, VerifiedSyncContribution, }; @@ -103,8 +102,7 @@ use state_processing::{ }, per_slot_processing, state_advance::{complete_state_advance, partial_state_advance}, - BlockSignatureStrategy, ConsensusContext, SigVerifiedOp, StateProcessingStrategy, - VerifyBlockRoot, VerifyOperation, + BlockSignatureStrategy, ConsensusContext, SigVerifiedOp, VerifyBlockRoot, VerifyOperation, }; use std::borrow::Cow; use std::cmp::Ordering; @@ -130,9 +128,6 @@ pub type ForkChoiceError = fork_choice::Error; /// Alias to appease clippy. type HashBlockTuple = (Hash256, RpcBlock); -/// The time-out before failure during an operation to take a read/write RwLock on the block -/// processing cache. -pub const BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT: Duration = Duration::from_secs(1); /// The time-out before failure during an operation to take a read/write RwLock on the /// attestation cache. pub const ATTESTATION_CACHE_LOCK_TIMEOUT: Duration = Duration::from_secs(1); @@ -176,6 +171,7 @@ pub const INVALID_FINALIZED_MERGE_TRANSITION_BLOCK_SHUTDOWN_REASON: &str = "Finalized merge transition block is invalid."; /// Defines the behaviour when a block/block-root for a skipped slot is requested. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum WhenSlotSkipped { /// If the slot is a skip slot, return `None`. /// @@ -450,8 +446,6 @@ pub struct BeaconChain { pub event_handler: Option>, /// Used to track the heads of the beacon chain. pub(crate) head_tracker: Arc, - /// A cache dedicated to block processing. - pub(crate) snapshot_cache: TimeoutRwLock>, /// Caches the attester shuffling for a given epoch and shuffling key root. pub shuffling_cache: TimeoutRwLock, /// A cache of eth1 deposit data at epoch boundaries for deposit finalization @@ -492,11 +486,6 @@ pub struct BeaconChain { pub data_availability_checker: Arc>, /// The KZG trusted setup used by this chain. pub kzg: Option>, - /// State with complete tree hash cache, ready for block production. - /// - /// NB: We can delete this once we have tree-states. - #[allow(clippy::type_complexity)] - pub block_production_state: Arc)>>>, } pub enum BeaconBlockResponseWrapper { @@ -773,7 +762,7 @@ impl BeaconChain { let iter = self.store.forwards_block_roots_iterator( start_slot, - local_head.beacon_state.clone_with(CloneConfig::none()), + local_head.beacon_state.clone(), local_head.beacon_block_root, &self.spec, )?; @@ -803,12 +792,7 @@ impl BeaconChain { let iter = self.store.forwards_block_roots_iterator_until( start_slot, end_slot, - || { - Ok(( - head.beacon_state.clone_with_only_committee_caches(), - head.beacon_block_root, - )) - }, + || Ok((head.beacon_state.clone(), head.beacon_block_root)), &self.spec, )?; Ok(iter @@ -879,7 +863,7 @@ impl BeaconChain { let iter = self.store.forwards_state_roots_iterator( start_slot, local_head.beacon_state_root(), - local_head.beacon_state.clone_with(CloneConfig::none()), + local_head.beacon_state.clone(), &self.spec, )?; @@ -900,12 +884,7 @@ impl BeaconChain { let iter = self.store.forwards_state_roots_iterator_until( start_slot, end_slot, - || { - Ok(( - head.beacon_state.clone_with_only_committee_caches(), - head.beacon_state_root(), - )) - }, + || Ok((head.beacon_state.clone(), head.beacon_state_root())), &self.spec, )?; Ok(iter @@ -3561,29 +3540,6 @@ impl BeaconChain { }); } - self.snapshot_cache - .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) - .ok_or(Error::SnapshotCacheLockTimeout) - .map(|mut snapshot_cache| { - snapshot_cache.insert( - BeaconSnapshot { - beacon_state: state, - beacon_block: signed_block.clone(), - beacon_block_root: block_root, - }, - None, - &self.spec, - ) - }) - .unwrap_or_else(|e| { - error!( - self.log, - "Failed to insert snapshot"; - "error" => ?e, - "task" => "process block" - ); - }); - self.head_tracker .register_block(block_root, parent_root, slot); @@ -4145,22 +4101,22 @@ impl BeaconChain { self.wait_for_fork_choice_before_block_production(slot)?; drop(fork_choice_timer); - // Producing a block requires the tree hash cache, so clone a full state corresponding to - // the head from the snapshot cache. Unfortunately we can't move the snapshot out of the - // cache (which would be fast), because we need to re-process the block after it has been - // signed. If we miss the cache or we're producing a block that conflicts with the head, - // fall back to getting the head from `slot - 1`. let state_load_timer = metrics::start_timer(&metrics::BLOCK_PRODUCTION_STATE_LOAD_TIMES); // Atomically read some values from the head whilst avoiding holding cached head `Arc` any // longer than necessary. - let (head_slot, head_block_root) = { + let (head_slot, head_block_root, head_state_root) = { let head = self.canonical_head.cached_head(); - (head.head_slot(), head.head_block_root()) + ( + head.head_slot(), + head.head_block_root(), + head.head_state_root(), + ) }; let (state, state_root_opt) = if head_slot < slot { // Attempt an aggressive re-org if configured and the conditions are right. - if let Some(re_org_state) = self.get_state_for_re_org(slot, head_slot, head_block_root) + if let Some((re_org_state, re_org_state_root)) = + self.get_state_for_re_org(slot, head_slot, head_block_root) { info!( self.log, @@ -4168,37 +4124,16 @@ impl BeaconChain { "slot" => slot, "head_to_reorg" => %head_block_root, ); - (re_org_state.pre_state, re_org_state.state_root) - } - // Normal case: proposing a block atop the current head using the cache. - else if let Some((_, cached_state)) = - self.get_state_from_block_production_cache(head_block_root) - { - (cached_state.pre_state, cached_state.state_root) - } - // Fall back to a direct read of the snapshot cache. - else if let Some(pre_state) = - self.get_state_from_snapshot_cache_for_block_production(head_block_root) - { - warn!( - self.log, - "Block production cache miss"; - "message" => "falling back to snapshot cache clone", - "slot" => slot - ); - (pre_state.pre_state, pre_state.state_root) + (re_org_state, Some(re_org_state_root)) } else { - warn!( - self.log, - "Block production cache miss"; - "message" => "this block is more likely to be orphaned", - "slot" => slot, - ); - let state = self - .state_at_slot(slot - 1, StateSkipConfig::WithStateRoots) - .map_err(|_| BlockProductionError::UnableToProduceAtSlot(slot))?; - - (state, None) + // Fetch the head state advanced through to `slot`, which should be present in the + // state cache thanks to the state advance timer. + let (state_root, state) = self + .store + .get_advanced_hot_state(head_block_root, slot, head_state_root) + .map_err(BlockProductionError::FailedToLoadState)? + .ok_or(BlockProductionError::UnableToProduceAtSlot(slot))?; + (state, Some(state_root)) } } else { warn!( @@ -4219,40 +4154,6 @@ impl BeaconChain { Ok((state, state_root_opt)) } - /// Get the state cached for block production *if* it matches `head_block_root`. - /// - /// This will clear the cache regardless of whether the block root matches, so only call this if - /// you think the `head_block_root` is likely to match! - fn get_state_from_block_production_cache( - &self, - head_block_root: Hash256, - ) -> Option<(Hash256, BlockProductionPreState)> { - // Take care to drop the lock as quickly as possible. - let mut lock = self.block_production_state.lock(); - let result = lock - .take() - .filter(|(cached_block_root, _)| *cached_block_root == head_block_root); - drop(lock); - result - } - - /// Get a state for block production from the snapshot cache. - fn get_state_from_snapshot_cache_for_block_production( - &self, - head_block_root: Hash256, - ) -> Option> { - if let Some(lock) = self - .snapshot_cache - .try_read_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) - { - let result = lock.get_state_for_block_production(head_block_root); - drop(lock); - result - } else { - None - } - } - /// Fetch the beacon state to use for producing a block if a 1-slot proposer re-org is viable. /// /// This function will return `None` if proposer re-orgs are disabled. @@ -4261,7 +4162,7 @@ impl BeaconChain { slot: Slot, head_slot: Slot, canonical_head: Hash256, - ) -> Option> { + ) -> Option<(BeaconState, Hash256)> { let re_org_head_threshold = self.config.re_org_head_threshold?; let re_org_parent_threshold = self.config.re_org_parent_threshold?; @@ -4345,26 +4246,14 @@ impl BeaconChain { drop(proposer_head_timer); let re_org_parent_block = proposer_head.parent_node.root; - // Only attempt a re-org if we hit the block production cache or snapshot cache. - let pre_state = self - .get_state_from_block_production_cache(re_org_parent_block) - .map(|(_, state)| state) + let (state_root, state) = self + .store + .get_advanced_hot_state_from_cache(re_org_parent_block, slot) .or_else(|| { warn!( - self.log, - "Block production cache miss"; - "message" => "falling back to snapshot cache during re-org", - "slot" => slot, - "block_root" => ?re_org_parent_block - ); - self.get_state_from_snapshot_cache_for_block_production(re_org_parent_block) - }) - .or_else(|| { - debug!( self.log, "Not attempting re-org"; - "reason" => "missed snapshot cache", - "parent_block" => ?re_org_parent_block, + "reason" => "no state in cache" ); None })?; @@ -4378,7 +4267,7 @@ impl BeaconChain { "threshold_weight" => proposer_head.re_org_head_weight_threshold ); - Some(pre_state) + Some((state, state_root)) } /// Get the proposer index and `prev_randao` value for a proposal at slot `proposal_slot`. @@ -4506,20 +4395,6 @@ impl BeaconChain { let (unadvanced_state, unadvanced_state_root) = if cached_head.head_block_root() == parent_block_root { (Cow::Borrowed(head_state), cached_head.head_state_root()) - } else if let Some(snapshot) = self - .snapshot_cache - .try_read_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) - .ok_or(Error::SnapshotCacheLockTimeout)? - .get_cloned(parent_block_root, CloneConfig::none()) - { - debug!( - self.log, - "Hit snapshot cache during withdrawals calculation"; - "slot" => proposal_slot, - "parent_block_root" => ?parent_block_root, - ); - let state_root = snapshot.beacon_state_root(); - (Cow::Owned(snapshot.beacon_state), state_root) } else { info!( self.log, @@ -4530,10 +4405,11 @@ impl BeaconChain { let block = self .get_blinded_block(&parent_block_root)? .ok_or(Error::MissingBeaconBlock(parent_block_root))?; - let state = self - .get_state(&block.state_root(), Some(block.slot()))? + let (state_root, state) = self + .store + .get_advanced_hot_state(parent_block_root, proposal_slot, block.state_root())? .ok_or(Error::MissingBeaconState(block.state_root()))?; - (Cow::Owned(state), block.state_root()) + (Cow::Owned(state), state_root) }; // Parent state epoch is the same as the proposal, we don't need to advance because the @@ -4910,6 +4786,7 @@ impl BeaconChain { drop(slot_timer); state.build_committee_cache(RelativeEpoch::Current, &self.spec)?; + state.apply_pending_mutations()?; let parent_root = if state.slot() > 0 { *state @@ -5383,7 +5260,6 @@ impl BeaconChain { &mut state, &block, signature_strategy, - StateProcessingStrategy::Accurate, VerifyBlockRoot::True, &mut ctxt, &self.spec, @@ -6308,6 +6184,17 @@ impl BeaconChain { "head_block_root" => head_block_root.to_string(), ); + // If the block's state will be so far ahead of `shuffling_epoch` that even its + // previous epoch committee cache will be too new, then error. Callers of this function + // shouldn't be requesting such old shufflings for this `head_block_root`. + let head_block_epoch = head_block.slot.epoch(T::EthSpec::slots_per_epoch()); + if head_block_epoch > shuffling_epoch + 1 { + return Err(Error::InvalidStateForShuffling { + state_epoch: head_block_epoch, + shuffling_epoch, + }); + } + let state_read_timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_STATE_READ_TIMES); @@ -6318,71 +6205,52 @@ impl BeaconChain { // to copy the head is liable to race-conditions. let head_state_opt = self.with_head(|head| { if head.beacon_block_root == head_block_root { - Ok(Some(( - head.beacon_state - .clone_with(CloneConfig::committee_caches_only()), - head.beacon_state_root(), - ))) + Ok(Some((head.beacon_state.clone(), head.beacon_state_root()))) } else { Ok::<_, Error>(None) } })?; + // Compute the `target_slot` to advance the block's state to. + // + // Since there's a one-epoch look-ahead on the attester shuffling, it suffices to + // only advance into the first slot of the epoch prior to `shuffling_epoch`. + // + // If the `head_block` is already ahead of that slot, then we should load the state + // at that slot, as we've determined above that the `shuffling_epoch` cache will + // not be too far in the past. + let target_slot = std::cmp::max( + shuffling_epoch + .saturating_sub(1_u64) + .start_slot(T::EthSpec::slots_per_epoch()), + head_block.slot, + ); + // If the head state is useful for this request, use it. Otherwise, read a state from - // disk. + // disk that is advanced as close as possible to `target_slot`. let (mut state, state_root) = if let Some((state, state_root)) = head_state_opt { (state, state_root) } else { - let block_state_root = head_block.state_root; - let max_slot = shuffling_epoch.start_slot(T::EthSpec::slots_per_epoch()); let (state_root, state) = self .store - .get_inconsistent_state_for_attestation_verification_only( - &head_block_root, - max_slot, - block_state_root, - )? - .ok_or(Error::MissingBeaconState(block_state_root))?; + .get_advanced_hot_state(head_block_root, target_slot, head_block.state_root)? + .ok_or(Error::MissingBeaconState(head_block.state_root))?; (state, state_root) }; - /* - * IMPORTANT - * - * Since it's possible that - * `Store::get_inconsistent_state_for_attestation_verification_only` was used to obtain - * the state, we cannot rely upon the following fields: - * - * - `state.state_roots` - * - `state.block_roots` - * - * These fields should not be used for the rest of this function. - */ - metrics::stop_timer(state_read_timer); let state_skip_timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_STATE_SKIP_TIMES); - // If the state is in an earlier epoch, advance it. If it's from a later epoch, reject - // it. + // If the state is still in an earlier epoch, advance it to the `target_slot` so + // that its next epoch committee cache matches the `shuffling_epoch`. if state.current_epoch() + 1 < shuffling_epoch { - // Since there's a one-epoch look-ahead on the attester shuffling, it suffices to - // only advance into the slot prior to the `shuffling_epoch`. - let target_slot = shuffling_epoch - .saturating_sub(1_u64) - .start_slot(T::EthSpec::slots_per_epoch()); - - // Advance the state into the required slot, using the "partial" method since the state - // roots are not relevant for the shuffling. + // Advance the state into the required slot, using the "partial" method since the + // state roots are not relevant for the shuffling. partial_state_advance(&mut state, Some(state_root), target_slot, &self.spec)?; - } else if state.current_epoch() > shuffling_epoch { - return Err(Error::InvalidStateForShuffling { - state_epoch: state.current_epoch(), - shuffling_epoch, - }); } - metrics::stop_timer(state_skip_timer); + let committee_building_timer = metrics::start_timer(&metrics::ATTESTATION_PROCESSING_COMMITTEE_BUILDING_TIMES); @@ -6391,8 +6259,7 @@ impl BeaconChain { state.build_committee_cache(relative_epoch, &self.spec)?; - let committee_cache = state.take_committee_cache(relative_epoch)?; - let committee_cache = Arc::new(committee_cache); + let committee_cache = state.committee_cache(relative_epoch)?.clone(); let shuffling_decision_block = shuffling_id.shuffling_decision_block; self.shuffling_cache @@ -6412,29 +6279,27 @@ impl BeaconChain { /// /// This could be a very expensive operation and should only be done in testing/analysis /// activities. + /// + /// This dump function previously used a backwards iterator but has been swapped to a forwards + /// iterator as it allows for MUCH better caching and rebasing. Memory usage of some tests went + /// from 5GB per test to 90MB. #[allow(clippy::type_complexity)] pub fn chain_dump( &self, ) -> Result>>, Error> { let mut dump = vec![]; - let mut last_slot = { - let head = self.canonical_head.cached_head(); - BeaconSnapshot { - beacon_block: Arc::new(head.snapshot.beacon_block.clone_as_blinded()), - beacon_block_root: head.snapshot.beacon_block_root, - beacon_state: head.snapshot.beacon_state.clone(), - } - }; - - dump.push(last_slot.clone()); + let mut prev_block_root = None; + let mut prev_beacon_state = None; - loop { - let beacon_block_root = last_slot.beacon_block.parent_root(); + for res in self.forwards_iter_block_roots(Slot::new(0))? { + let (beacon_block_root, _) = res?; - if beacon_block_root == Hash256::zero() { - break; // Genesis has been reached. + // Do not include snapshots at skipped slots. + if Some(beacon_block_root) == prev_block_root { + continue; } + prev_block_root = Some(beacon_block_root); let beacon_block = self .store @@ -6443,25 +6308,31 @@ impl BeaconChain { Error::DBInconsistent(format!("Missing block {}", beacon_block_root)) })?; let beacon_state_root = beacon_block.state_root(); - let beacon_state = self + + let mut beacon_state = self .store .get_state(&beacon_state_root, Some(beacon_block.slot()))? .ok_or_else(|| { Error::DBInconsistent(format!("Missing state {:?}", beacon_state_root)) })?; - let slot = BeaconSnapshot { + // This beacon state might come from the freezer DB, which means it could have pending + // updates or lots of untethered memory. We rebase it on the previous state in order to + // address this. + beacon_state.apply_pending_mutations()?; + if let Some(prev) = prev_beacon_state { + beacon_state.rebase_on(&prev, &self.spec)?; + } + beacon_state.build_caches(&self.spec)?; + prev_beacon_state = Some(beacon_state.clone()); + + let snapshot = BeaconSnapshot { beacon_block: Arc::new(beacon_block), beacon_block_root, beacon_state, }; - - dump.push(slot.clone()); - last_slot = slot; + dump.push(snapshot); } - - dump.reverse(); - Ok(dump) } @@ -6696,6 +6567,10 @@ impl BeaconChain { self.data_availability_checker.data_availability_boundary() } + pub fn logger(&self) -> &Logger { + &self.log + } + /// Gets the `LightClientBootstrap` object for a requested block root. /// /// Returns `None` when the state or block is not found in the database. diff --git a/beacon_node/beacon_chain/src/beacon_proposer_cache.rs b/beacon_node/beacon_chain/src/beacon_proposer_cache.rs index ca390712b13..d10bbfbbc5f 100644 --- a/beacon_node/beacon_chain/src/beacon_proposer_cache.rs +++ b/beacon_node/beacon_chain/src/beacon_proposer_cache.rs @@ -17,8 +17,7 @@ use std::cmp::Ordering; use std::num::NonZeroUsize; use types::non_zero_usize::new_non_zero_usize; use types::{ - BeaconState, BeaconStateError, ChainSpec, CloneConfig, Epoch, EthSpec, Fork, Hash256, Slot, - Unsigned, + BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, Fork, Hash256, Slot, Unsigned, }; /// The number of sets of proposer indices that should be cached. @@ -145,10 +144,7 @@ pub fn compute_proposer_duties_from_head( let (mut state, head_state_root, head_block_root) = { let head = chain.canonical_head.cached_head(); // Take a copy of the head state. - let head_state = head - .snapshot - .beacon_state - .clone_with(CloneConfig::committee_caches_only()); + let head_state = head.snapshot.beacon_state.clone(); let head_state_root = head.head_state_root(); let head_block_root = head.head_block_root(); (head_state, head_state_root, head_block_root) diff --git a/beacon_node/beacon_chain/src/beacon_snapshot.rs b/beacon_node/beacon_chain/src/beacon_snapshot.rs index afb13247766..e9fde48ac67 100644 --- a/beacon_node/beacon_chain/src/beacon_snapshot.rs +++ b/beacon_node/beacon_chain/src/beacon_snapshot.rs @@ -1,8 +1,8 @@ use serde::Serialize; use std::sync::Arc; use types::{ - beacon_state::CloneConfig, AbstractExecPayload, BeaconState, EthSpec, FullPayload, Hash256, - SignedBeaconBlock, + AbstractExecPayload, BeaconState, EthSpec, FullPayload, Hash256, SignedBeaconBlock, + SignedBlindedBeaconBlock, }; /// Represents some block and its associated state. Generally, this will be used for tracking the @@ -14,6 +14,19 @@ pub struct BeaconSnapshot = FullPayl pub beacon_state: BeaconState, } +/// This snapshot is to be used for verifying a child of `self.beacon_block`. +#[derive(Debug)] +pub struct PreProcessingSnapshot { + /// This state is equivalent to the `self.beacon_block.state_root()` state that has been + /// advanced forward one slot using `per_slot_processing`. This state is "primed and ready" for + /// the application of another block. + pub pre_state: BeaconState, + /// This value is only set to `Some` if the `pre_state` was *not* advanced forward. + pub beacon_state_root: Option, + pub beacon_block: SignedBlindedBeaconBlock, + pub beacon_block_root: Hash256, +} + impl> BeaconSnapshot { /// Create a new checkpoint. pub fn new( @@ -48,12 +61,4 @@ impl> BeaconSnapshot { self.beacon_block_root = beacon_block_root; self.beacon_state = beacon_state; } - - pub fn clone_with(&self, clone_config: CloneConfig) -> Self { - Self { - beacon_block: self.beacon_block.clone(), - beacon_block_root: self.beacon_block_root, - beacon_state: self.beacon_state.clone_with(clone_config), - } - } } diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index a1ae260d930..263b9f9e013 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -2,7 +2,7 @@ use derivative::Derivative; use slot_clock::SlotClock; use std::sync::Arc; -use crate::beacon_chain::{BeaconChain, BeaconChainTypes, BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT}; +use crate::beacon_chain::{BeaconChain, BeaconChainTypes}; use crate::block_verification::{ cheap_state_advance_to_obtain_committees, get_validator_pubkey_cache, process_block_slash_info, BlockSlashInfo, @@ -11,15 +11,13 @@ use crate::kzg_utils::{validate_blob, validate_blobs}; use crate::{metrics, BeaconChainError}; use kzg::{Error as KzgError, Kzg, KzgCommitment}; use merkle_proof::MerkleTreeError; -use slog::{debug, warn}; +use slog::debug; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; use std::time::Duration; use tree_hash::TreeHash; use types::blob_sidecar::BlobIdentifier; -use types::{ - BeaconStateError, BlobSidecar, CloneConfig, EthSpec, Hash256, SignedBeaconBlockHeader, Slot, -}; +use types::{BeaconStateError, BlobSidecar, EthSpec, Hash256, SignedBeaconBlockHeader, Slot}; /// An error occurred while validating a gossip blob. #[derive(Debug)] @@ -514,98 +512,43 @@ pub fn validate_blob_sidecar_for_gossip( "block_root" => %block_root, "index" => %blob_index, ); - if let Some(mut snapshot) = chain - .snapshot_cache - .try_read_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) - .and_then(|snapshot_cache| { - snapshot_cache.get_cloned(block_parent_root, CloneConfig::committee_caches_only()) - }) - { - if snapshot.beacon_state.slot() == blob_slot { - debug!( - chain.log, - "Cloning snapshot cache state for blob verification"; - "block_root" => %block_root, - "index" => %blob_index, - ); - ( - snapshot - .beacon_state - .get_beacon_proposer_index(blob_slot, &chain.spec)?, - snapshot.beacon_state.fork(), - ) - } else { - debug!( - chain.log, - "Cloning and advancing snapshot cache state for blob verification"; - "block_root" => %block_root, - "index" => %blob_index, - ); - let state = - cheap_state_advance_to_obtain_committees::<_, GossipBlobError>( - &mut snapshot.beacon_state, - Some(snapshot.beacon_block_root), - blob_slot, - &chain.spec, - )?; - ( - state.get_beacon_proposer_index(blob_slot, &chain.spec)?, - state.fork(), - ) - } - } - // Need to advance the state to get the proposer index - else { - warn!( - chain.log, - "Snapshot cache miss for blob verification"; - "block_root" => %block_root, - "index" => %blob_index, - ); - - let parent_block = chain - .get_blinded_block(&block_parent_root) - .map_err(GossipBlobError::BeaconChainError)? - .ok_or_else(|| { - GossipBlobError::from(BeaconChainError::MissingBeaconBlock(block_parent_root)) - })?; - - let mut parent_state = chain - .get_state(&parent_block.state_root(), Some(parent_block.slot()))? - .ok_or_else(|| { - BeaconChainError::DBInconsistent(format!( - "Missing state {:?}", - parent_block.state_root() - )) - })?; - let state = cheap_state_advance_to_obtain_committees::<_, GossipBlobError>( - &mut parent_state, - Some(parent_block.state_root()), - blob_slot, - &chain.spec, - )?; - - let proposers = state.get_beacon_proposer_indices(&chain.spec)?; - let proposer_index = *proposers - .get(blob_slot.as_usize() % T::EthSpec::slots_per_epoch() as usize) - .ok_or_else(|| BeaconChainError::NoProposerForSlot(blob_slot))?; - - let fork = state.fork(); - // Prime the proposer shuffling cache with the newly-learned value. - chain.beacon_proposer_cache.lock().insert( - blob_epoch, - proposer_shuffling_root, - proposers, - fork, - )?; - (proposer_index, fork) - } + let (parent_state_root, mut parent_state) = chain + .store + .get_advanced_hot_state(block_parent_root, blob_slot, parent_block.state_root) + .map_err(|e| GossipBlobError::BeaconChainError(e.into()))? + .ok_or_else(|| { + BeaconChainError::DBInconsistent(format!( + "Missing state for parent block {block_parent_root:?}", + )) + })?; + + let state = cheap_state_advance_to_obtain_committees::<_, GossipBlobError>( + &mut parent_state, + Some(parent_state_root), + blob_slot, + &chain.spec, + )?; + + let proposers = state.get_beacon_proposer_indices(&chain.spec)?; + let proposer_index = *proposers + .get(blob_slot.as_usize() % T::EthSpec::slots_per_epoch() as usize) + .ok_or_else(|| BeaconChainError::NoProposerForSlot(blob_slot))?; + + // Prime the proposer shuffling cache with the newly-learned value. + chain.beacon_proposer_cache.lock().insert( + blob_epoch, + proposer_shuffling_root, + proposers, + state.fork(), + )?; + (proposer_index, state.fork()) }; // Signature verify the signed block header. let signature_is_valid = { let pubkey_cache = get_validator_pubkey_cache(chain).map_err(|_| GossipBlobError::PubkeyCacheTimeout)?; + let pubkey = pubkey_cache .get(proposer_index) .ok_or_else(|| GossipBlobError::UnknownValidator(proposer_index as u64))?; diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 38648949f9c..866dde5a763 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -48,6 +48,7 @@ // returned alongside. #![allow(clippy::result_large_err)] +use crate::beacon_snapshot::PreProcessingSnapshot; use crate::blob_verification::{GossipBlobError, GossipVerifiedBlob}; use crate::block_verification_types::{ AsBlock, BlockContentsError, BlockImportData, GossipVerifiedBlockContents, RpcBlock, @@ -59,14 +60,10 @@ use crate::execution_payload::{ AllowOptimisticImport, NotifyExecutionLayer, PayloadNotifier, }; use crate::observed_block_producers::SeenBlock; -use crate::snapshot_cache::PreProcessingSnapshot; use crate::validator_monitor::HISTORIC_EPOCHS as VALIDATOR_MONITOR_HISTORIC_EPOCHS; use crate::validator_pubkey_cache::ValidatorPubkeyCache; use crate::{ - beacon_chain::{ - BeaconForkChoice, ForkChoiceError, BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT, - VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT, - }, + beacon_chain::{BeaconForkChoice, ForkChoiceError, VALIDATOR_PUBKEY_CACHE_LOCK_TIMEOUT}, metrics, BeaconChain, BeaconChainError, BeaconChainTypes, }; use derivative::Derivative; @@ -86,22 +83,21 @@ use state_processing::{ block_signature_verifier::{BlockSignatureVerifier, Error as BlockSignatureVerifierError}, per_block_processing, per_slot_processing, state_advance::partial_state_advance, - BlockProcessingError, BlockSignatureStrategy, ConsensusContext, SlotProcessingError, - StateProcessingStrategy, VerifyBlockRoot, + AllCaches, BlockProcessingError, BlockSignatureStrategy, ConsensusContext, SlotProcessingError, + VerifyBlockRoot, }; use std::borrow::Cow; use std::fmt::Debug; use std::fs; use std::io::Write; use std::sync::Arc; -use std::time::Duration; use store::{Error as DBError, HotStateSummary, KeyValueStore, StoreOp}; use task_executor::JoinHandle; use tree_hash::TreeHash; use types::{ - BeaconBlockRef, BeaconState, BeaconStateError, ChainSpec, CloneConfig, Epoch, EthSpec, - ExecutionBlockHash, Hash256, InconsistentFork, PublicKey, PublicKeyBytes, RelativeEpoch, - SignedBeaconBlock, SignedBeaconBlockHeader, Slot, + BeaconBlockRef, BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, ExecutionBlockHash, + Hash256, InconsistentFork, PublicKey, PublicKeyBytes, RelativeEpoch, SignedBeaconBlock, + SignedBeaconBlockHeader, Slot, }; use types::{BlobSidecar, ExecPayload}; @@ -578,7 +574,7 @@ pub fn signature_verify_chain_segment( } let (first_root, first_block) = chain_segment.remove(0); - let (mut parent, first_block) = load_parent(first_root, first_block, chain)?; + let (mut parent, first_block) = load_parent(first_block, chain)?; let slot = first_block.slot(); chain_segment.insert(0, (first_root, first_block)); @@ -893,7 +889,7 @@ impl GossipVerifiedBlock { } else { // The proposer index was *not* cached and we must load the parent in order to determine // the proposer index. - let (mut parent, block) = load_parent(block_root, block, chain)?; + let (mut parent, block) = load_parent(block, chain)?; debug!( chain.log, @@ -1042,7 +1038,7 @@ impl SignatureVerifiedBlock { // Check the anchor slot before loading the parent, to avoid spurious lookups. check_block_against_anchor_slot(block.message(), chain)?; - let (mut parent, block) = load_parent(block_root, block, chain)?; + let (mut parent, block) = load_parent(block, chain)?; let state = cheap_state_advance_to_obtain_committees::<_, BlockError>( &mut parent.pre_state, @@ -1092,7 +1088,7 @@ impl SignatureVerifiedBlock { let (mut parent, block) = if let Some(parent) = from.parent { (parent, from.block) } else { - load_parent(from.block_root, from.block, chain)? + load_parent(from.block, chain)? }; let state = cheap_state_advance_to_obtain_committees::<_, BlockError>( @@ -1154,7 +1150,7 @@ impl IntoExecutionPendingBlock for SignatureVerifiedBloc let (parent, block) = if let Some(parent) = self.parent { (parent, self.block) } else { - load_parent(self.block_root, self.block, chain) + load_parent(self.block, chain) .map_err(|e| BlockSlashInfo::SignatureValid(header.clone(), e))? }; @@ -1386,8 +1382,18 @@ impl ExecutionPendingBlock { let catchup_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_CATCHUP_STATE); // Stage a batch of operations to be completed atomically if this block is imported - // successfully. - let mut confirmed_state_roots = vec![]; + // successfully. We include the state root of the pre-state, which may be an advanced state + // that was stored in the DB with a `temporary` flag. + let mut state = parent.pre_state; + + let mut confirmed_state_roots = if state.slot() > parent.beacon_block.slot() { + // Advanced pre-state. Delete its temporary flag. + let pre_state_root = state.update_tree_hash_cache()?; + vec![pre_state_root] + } else { + // Pre state is parent state. It is already stored in the DB without temporary status. + vec![] + }; // The block must have a higher slot than its parent. if block.slot() <= parent.beacon_block.slot() { @@ -1397,14 +1403,6 @@ impl ExecutionPendingBlock { }); } - let mut summaries = vec![]; - - // Transition the parent state to the block slot. - // - // It is important to note that we're using a "pre-state" here, one that has potentially - // been advanced one slot forward from `parent.beacon_block.slot`. - let mut state = parent.pre_state; - // Perform a sanity check on the pre-state. let parent_slot = parent.beacon_block.slot(); if state.slot() < parent_slot || state.slot() > block.slot() { @@ -1423,6 +1421,12 @@ impl ExecutionPendingBlock { eth1_deposit_index: state.eth1_deposit_index(), }; + // Transition the parent state to the block slot. + // + // It is important to note that we're using a "pre-state" here, one that has potentially + // been advanced one slot forward from `parent.beacon_block.slot`. + let mut summaries = vec![]; + let distance = block.slot().as_u64().saturating_sub(state.slot().as_u64()); for _ in 0..distance { let state_root = if parent.beacon_block.slot() == state.slot() { @@ -1522,8 +1526,7 @@ impl ExecutionPendingBlock { let committee_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_COMMITTEE); - state.build_committee_cache(RelativeEpoch::Previous, &chain.spec)?; - state.build_committee_cache(RelativeEpoch::Current, &chain.spec)?; + state.build_all_committee_caches(&chain.spec)?; metrics::stop_timer(committee_timer); @@ -1564,7 +1567,6 @@ impl ExecutionPendingBlock { block.as_block(), // Signatures were verified earlier in this function. BlockSignatureStrategy::NoVerification, - StateProcessingStrategy::Accurate, VerifyBlockRoot::True, &mut consensus_context, &chain.spec, @@ -1839,12 +1841,9 @@ fn verify_parent_block_is_known( /// whilst attempting the operation. #[allow(clippy::type_complexity)] fn load_parent>( - block_root: Hash256, block: B, chain: &BeaconChain, ) -> Result<(PreProcessingSnapshot, B), BlockError> { - let spec = &chain.spec; - // Reject any block if its parent is not known to fork choice. // // A block that is not in fork choice is either: @@ -1863,45 +1862,10 @@ fn load_parent>( return Err(BlockError::ParentUnknown(block.into_rpc_block())); } - let block_delay = chain - .block_times_cache - .read() - .get_block_delays( - block_root, - chain - .slot_clock - .start_of(block.slot()) - .unwrap_or_else(|| Duration::from_secs(0)), - ) - .observed; - let db_read_timer = metrics::start_timer(&metrics::BLOCK_PROCESSING_DB_READ); - let result = if let Some((snapshot, cloned)) = chain - .snapshot_cache - .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) - .and_then(|mut snapshot_cache| { - snapshot_cache.get_state_for_block_processing( - block.parent_root(), - block.slot(), - block_delay, - spec, - ) - }) { - if cloned { - metrics::inc_counter(&metrics::BLOCK_PROCESSING_SNAPSHOT_CACHE_CLONES); - debug!( - chain.log, - "Cloned snapshot for late block/skipped slot"; - "slot" => %block.slot(), - "parent_slot" => %snapshot.beacon_block.slot(), - "parent_root" => ?block.parent_root(), - "block_delay" => ?block_delay, - ); - } - Ok((snapshot, block)) - } else { - // Load the blocks parent block from the database, returning invalid if that block is not + let result = { + // Load the block's parent block from the database, returning invalid if that block is not // found. // // We don't return a DBInconsistent error here since it's possible for a block to @@ -1925,7 +1889,7 @@ fn load_parent>( // Retrieve any state that is advanced through to at most `block.slot()`: this is // particularly important if `block` descends from the finalized/split block, but at a slot // prior to the finalized slot (which is invalid and inaccessible in our DB schema). - let (parent_state_root, parent_state) = chain + let (parent_state_root, state) = chain .store .get_advanced_hot_state(root, block.slot(), parent_block.state_root())? .ok_or_else(|| { @@ -1934,22 +1898,46 @@ fn load_parent>( ) })?; - metrics::inc_counter(&metrics::BLOCK_PROCESSING_SNAPSHOT_CACHE_MISSES); - debug!( - chain.log, - "Missed snapshot cache"; - "slot" => block.slot(), - "parent_slot" => parent_block.slot(), - "parent_root" => ?block.parent_root(), - "block_delay" => ?block_delay, - ); + if !state.all_caches_built() { + debug!( + chain.log, + "Parent state lacks built caches"; + "block_slot" => block.slot(), + "state_slot" => state.slot(), + ); + } + + if block.slot() != state.slot() { + debug!( + chain.log, + "Parent state is not advanced"; + "block_slot" => block.slot(), + "state_slot" => state.slot(), + ); + } + + let beacon_state_root = if state.slot() == parent_block.slot() { + // Sanity check. + if parent_state_root != parent_block.state_root() { + return Err(BeaconChainError::DBInconsistent(format!( + "Parent state at slot {} has the wrong state root: {:?} != {:?}", + state.slot(), + parent_state_root, + parent_block.state_root() + )) + .into()); + } + Some(parent_block.state_root()) + } else { + None + }; Ok(( PreProcessingSnapshot { beacon_block: parent_block, beacon_block_root: root, - pre_state: parent_state, - beacon_state_root: Some(parent_state_root), + pre_state: state, + beacon_state_root, }, block, )) @@ -2030,7 +2018,7 @@ pub fn cheap_state_advance_to_obtain_committees<'a, E: EthSpec, Err: BlockBlobEr } else if state.slot() > block_slot { Err(Err::not_later_than_parent_error(block_slot, state.slot())) } else { - let mut state = state.clone_with(CloneConfig::committee_caches_only()); + let mut state = state.clone(); let target_slot = block_epoch.start_slot(E::slots_per_epoch()); // Advance the state into the same epoch as the block. Use the "partial" method since state diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 56381a7806f..be6b1f9b2bf 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -12,7 +12,6 @@ use crate::light_client_server_cache::LightClientServerCache; use crate::migrate::{BackgroundMigrator, MigratorConfig}; use crate::persisted_beacon_chain::PersistedBeaconChain; use crate::shuffling_cache::{BlockShufflingIds, ShufflingCache}; -use crate::snapshot_cache::SnapshotCache; use crate::timeout_rw_lock::TimeoutRwLock; use crate::validator_monitor::{ValidatorMonitor, ValidatorMonitorConfig}; use crate::validator_pubkey_cache::ValidatorPubkeyCache; @@ -32,7 +31,7 @@ use proto_array::{DisallowedReOrgOffsets, ReOrgThreshold}; use slasher::Slasher; use slog::{crit, debug, error, info, o, Logger}; use slot_clock::{SlotClock, TestingSlotClock}; -use state_processing::per_slot_processing; +use state_processing::{per_slot_processing, AllCaches}; use std::marker::PhantomData; use std::sync::Arc; use std::time::Duration; @@ -462,7 +461,7 @@ where // Prime all caches before storing the state in the database and computing the tree hash // root. weak_subj_state - .build_caches(&self.spec) + .build_all_caches(&self.spec) .map_err(|e| format!("Error building caches on checkpoint state: {e:?}"))?; let weak_subj_state_root = weak_subj_state .update_tree_hash_cache() @@ -537,6 +536,13 @@ where // Write the state, block and blobs non-atomically, it doesn't matter if they're forgotten // about on a crash restart. + store + .update_finalized_state( + weak_subj_state_root, + weak_subj_block_root, + weak_subj_state.clone(), + ) + .map_err(|e| format!("Failed to set checkpoint state as finalized state: {:?}", e))?; store .put_state(&weak_subj_state_root, &weak_subj_state) .map_err(|e| format!("Failed to store weak subjectivity state: {e:?}"))?; @@ -851,10 +857,8 @@ where let genesis_validators_root = head_snapshot.beacon_state.genesis_validators_root(); let genesis_time = head_snapshot.beacon_state.genesis_time(); - let head_for_snapshot_cache = head_snapshot.clone(); let canonical_head = CanonicalHead::new(fork_choice, Arc::new(head_snapshot)); let shuffling_cache_size = self.chain_config.shuffling_cache_size; - let snapshot_cache_size = self.chain_config.snapshot_cache_size; // Calculate the weak subjectivity point in which to backfill blocks to. let genesis_backfill_slot = if self.chain_config.genesis_backfill { @@ -930,10 +934,6 @@ where fork_choice_signal_rx, event_handler: self.event_handler, head_tracker, - snapshot_cache: TimeoutRwLock::new(SnapshotCache::new( - snapshot_cache_size, - head_for_snapshot_cache, - )), shuffling_cache: TimeoutRwLock::new(ShufflingCache::new( shuffling_cache_size, head_shuffling_ids, @@ -962,7 +962,6 @@ where .map_err(|e| format!("Error initializing DataAvailabiltyChecker: {:?}", e))?, ), kzg: self.kzg.clone(), - block_production_state: Arc::new(Mutex::new(None)), }; let head = beacon_chain.head_snapshot(); diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index 734575d2c0d..a84cfab298d 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -35,10 +35,7 @@ use crate::beacon_chain::ATTESTATION_CACHE_LOCK_TIMEOUT; use crate::persisted_fork_choice::PersistedForkChoice; use crate::shuffling_cache::BlockShufflingIds; use crate::{ - beacon_chain::{ - BeaconForkChoice, BeaconStore, OverrideForkchoiceUpdate, - BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT, FORK_CHOICE_DB_KEY, - }, + beacon_chain::{BeaconForkChoice, BeaconStore, OverrideForkchoiceUpdate, FORK_CHOICE_DB_KEY}, block_times_cache::BlockTimesCache, events::ServerSentEventHandler, metrics, @@ -54,6 +51,7 @@ use itertools::process_results; use parking_lot::{Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard}; use slog::{crit, debug, error, warn, Logger}; use slot_clock::SlotClock; +use state_processing::AllCaches; use std::sync::Arc; use std::time::Duration; use store::{iter::StateRootsIterator, KeyValueStoreOp, StoreItem}; @@ -466,9 +464,7 @@ impl BeaconChain { pub fn head_beacon_state_cloned(&self) -> BeaconState { // Don't clone whilst holding the read-lock, take an Arc-clone to reduce lock contention. let snapshot: Arc<_> = self.head_snapshot(); - snapshot - .beacon_state - .clone_with(CloneConfig::committee_caches_only()) + snapshot.beacon_state.clone() } /// Execute the fork choice algorithm and enthrone the result as the canonical head. @@ -652,48 +648,31 @@ impl BeaconChain { let new_cached_head = if new_view.head_block_root != old_view.head_block_root { metrics::inc_counter(&metrics::FORK_CHOICE_CHANGED_HEAD); - // Try and obtain the snapshot for `beacon_block_root` from the snapshot cache, falling - // back to a database read if that fails. - let new_snapshot = self - .snapshot_cache - .try_read_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) - .and_then(|snapshot_cache| { - snapshot_cache.get_cloned( + let mut new_snapshot = { + let beacon_block = self + .store + .get_full_block(&new_view.head_block_root)? + .ok_or(Error::MissingBeaconBlock(new_view.head_block_root))?; + + let (_, beacon_state) = self + .store + .get_advanced_hot_state( new_view.head_block_root, - CloneConfig::committee_caches_only(), - ) - }) - .map::, _>(Ok) - .unwrap_or_else(|| { - let beacon_block = self - .store - .get_full_block(&new_view.head_block_root)? - .ok_or(Error::MissingBeaconBlock(new_view.head_block_root))?; - - let (_, beacon_state) = self - .store - .get_advanced_hot_state( - new_view.head_block_root, - current_slot, - beacon_block.state_root(), - )? - .ok_or(Error::MissingBeaconState(beacon_block.state_root()))?; - - Ok(BeaconSnapshot { - beacon_block: Arc::new(beacon_block), - beacon_block_root: new_view.head_block_root, - beacon_state, - }) - }) - .and_then(|mut snapshot| { - // Regardless of where we got the state from, attempt to build the committee - // caches. - snapshot - .beacon_state - .build_all_committee_caches(&self.spec) - .map_err(Into::into) - .map(|()| snapshot) - })?; + current_slot, + beacon_block.state_root(), + )? + .ok_or(Error::MissingBeaconState(beacon_block.state_root()))?; + + BeaconSnapshot { + beacon_block: Arc::new(beacon_block), + beacon_block_root: new_view.head_block_root, + beacon_state, + } + }; + + // Regardless of where we got the state from, attempt to build all the + // caches except the tree hash cache. + new_snapshot.beacon_state.build_all_caches(&self.spec)?; let new_cached_head = CachedHead { snapshot: Arc::new(new_snapshot), @@ -834,25 +813,6 @@ impl BeaconChain { .beacon_state .attester_shuffling_decision_root(self.genesis_block_root, RelativeEpoch::Current); - // Update the snapshot cache with the latest head value. - // - // This *could* be done inside `recompute_head`, however updating the head on the snapshot - // cache is not critical so we avoid placing it on a critical path. Note that this function - // will not return an error if the update fails, it will just log an error. - self.snapshot_cache - .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) - .map(|mut snapshot_cache| { - snapshot_cache.update_head(new_snapshot.beacon_block_root); - }) - .unwrap_or_else(|| { - error!( - self.log, - "Failed to obtain cache write lock"; - "lock" => "snapshot_cache", - "task" => "update head" - ); - }); - match BlockShufflingIds::try_from_head( new_snapshot.beacon_block_root, &new_snapshot.beacon_state, @@ -998,26 +958,6 @@ impl BeaconChain { .start_slot(T::EthSpec::slots_per_epoch()), ); - self.snapshot_cache - .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) - .map(|mut snapshot_cache| { - snapshot_cache.prune(new_view.finalized_checkpoint.epoch); - debug!( - self.log, - "Snapshot cache pruned"; - "new_len" => snapshot_cache.len(), - "remaining_roots" => ?snapshot_cache.beacon_block_roots(), - ); - }) - .unwrap_or_else(|| { - error!( - self.log, - "Failed to obtain cache write lock"; - "lock" => "snapshot_cache", - "task" => "prune" - ); - }); - self.attester_cache .prune_below(new_view.finalized_checkpoint.epoch); diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index 0772aff6710..255b8f00497 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -75,8 +75,6 @@ pub struct ChainConfig { pub optimistic_finalized_sync: bool, /// The size of the shuffling cache, pub shuffling_cache_size: usize, - /// The size of the snapshot cache. - pub snapshot_cache_size: usize, /// If using a weak-subjectivity sync, whether we should download blocks all the way back to /// genesis. pub genesis_backfill: bool, @@ -116,7 +114,6 @@ impl Default for ChainConfig { // This value isn't actually read except in tests. optimistic_finalized_sync: true, shuffling_cache_size: crate::shuffling_cache::DEFAULT_CACHE_SIZE, - snapshot_cache_size: crate::snapshot_cache::DEFAULT_SNAPSHOT_CACHE_SIZE, genesis_backfill: false, always_prepare_payload: false, epochs_per_migration: crate::migrate::DEFAULT_EPOCHS_PER_MIGRATION, diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index f4c1bc308c0..3a8bccbad5c 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -1059,7 +1059,7 @@ mod test { let chain = &harness.chain; let log = chain.log.clone(); let head = chain.head_snapshot(); - let parent_state = head.beacon_state.clone_with_only_committee_caches(); + let parent_state = head.beacon_state.clone(); let target_slot = chain.slot().expect("should get slot") + 1; let parent_root = head.beacon_block_root; diff --git a/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs index b6dbf2b952f..f8a243bd9e8 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs @@ -8,7 +8,7 @@ use crate::{ use lru::LruCache; use parking_lot::RwLock; use ssz_derive::{Decode, Encode}; -use state_processing::{BlockReplayer, StateProcessingStrategy}; +use state_processing::BlockReplayer; use std::sync::Arc; use store::OnDiskConsensusContext; use types::beacon_block_body::KzgCommitments; @@ -189,7 +189,6 @@ impl StateLRUCache { let block_replayer: BlockReplayer<'_, T::EthSpec, AvailabilityCheckError, _> = BlockReplayer::new(parent_state, &self.spec) .no_signature_verification() - .state_processing_strategy(StateProcessingStrategy::Accurate) .state_root_iter(state_roots.into_iter()) .minimal_block_root_verification(); diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 9c82e964cc0..340f1f9f797 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -31,6 +31,7 @@ use state_processing::{ use std::time::Duration; use task_executor::ShutdownReason; use tokio::task::JoinError; +use types::milhouse::Error as MilhouseError; use types::*; macro_rules! easy_from_to { @@ -224,6 +225,7 @@ pub enum BeaconChainError { AvailabilityCheckError(AvailabilityCheckError), LightClientError(LightClientError), UnsupportedFork, + MilhouseError(MilhouseError), } easy_from_to!(SlotProcessingError, BeaconChainError); @@ -253,6 +255,7 @@ easy_from_to!(InconsistentFork, BeaconChainError); easy_from_to!(AvailabilityCheckError, BeaconChainError); easy_from_to!(EpochCacheError, BeaconChainError); easy_from_to!(LightClientError, BeaconChainError); +easy_from_to!(MilhouseError, BeaconChainError); #[derive(Debug)] pub enum BlockProductionError { @@ -279,6 +282,7 @@ pub enum BlockProductionError { TerminalPoWBlockLookupFailed(execution_layer::Error), GetPayloadFailed(execution_layer::Error), FailedToReadFinalizedBlock(store::Error), + FailedToLoadState(store::Error), MissingFinalizedBlock(Hash256), BlockTooLarge(usize), ShuttingDown, diff --git a/beacon_node/beacon_chain/src/eth1_chain.rs b/beacon_node/beacon_chain/src/eth1_chain.rs index 3ec39f9d192..31297244e3e 100644 --- a/beacon_node/beacon_chain/src/eth1_chain.rs +++ b/beacon_node/beacon_chain/src/eth1_chain.rs @@ -1020,6 +1020,7 @@ mod test { mod collect_valid_votes { use super::*; + use types::List; fn get_eth1_data_vec(n: u64, block_number_offset: u64) -> Vec<(Eth1Data, BlockNumber)> { (0..n) @@ -1067,12 +1068,14 @@ mod test { let votes_to_consider = get_eth1_data_vec(slots, 0); - *state.eth1_data_votes_mut() = votes_to_consider[0..slots as usize / 4] - .iter() - .map(|(eth1_data, _)| eth1_data) - .cloned() - .collect::>() - .into(); + *state.eth1_data_votes_mut() = List::new( + votes_to_consider[0..slots as usize / 4] + .iter() + .map(|(eth1_data, _)| eth1_data) + .cloned() + .collect::>(), + ) + .unwrap(); let votes = collect_valid_votes(&state, &votes_to_consider.clone().into_iter().collect()); @@ -1096,12 +1099,14 @@ mod test { .expect("should have some eth1 data") .clone(); - *state.eth1_data_votes_mut() = vec![duplicate_eth1_data.clone(); 4] - .iter() - .map(|(eth1_data, _)| eth1_data) - .cloned() - .collect::>() - .into(); + *state.eth1_data_votes_mut() = List::new( + vec![duplicate_eth1_data.clone(); 4] + .iter() + .map(|(eth1_data, _)| eth1_data) + .cloned() + .collect::>(), + ) + .unwrap(); let votes = collect_valid_votes(&state, &votes_to_consider.into_iter().collect()); assert_votes!( diff --git a/beacon_node/beacon_chain/src/fork_revert.rs b/beacon_node/beacon_chain/src/fork_revert.rs index 084ae95e096..8d1c29f46f6 100644 --- a/beacon_node/beacon_chain/src/fork_revert.rs +++ b/beacon_node/beacon_chain/src/fork_revert.rs @@ -5,7 +5,7 @@ use slog::{info, warn, Logger}; use state_processing::state_advance::complete_state_advance; use state_processing::{ per_block_processing, per_block_processing::BlockSignatureStrategy, ConsensusContext, - StateProcessingStrategy, VerifyBlockRoot, + VerifyBlockRoot, }; use std::sync::Arc; use std::time::Duration; @@ -175,7 +175,6 @@ pub fn reset_fork_choice_to_finalization, Cold: It &mut state, &block, BlockSignatureStrategy::NoVerification, - StateProcessingStrategy::Accurate, VerifyBlockRoot::True, &mut ctxt, spec, diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 71c96d0fd55..c1df9ede87a 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -51,7 +51,6 @@ mod pre_finalization_cache; pub mod proposer_prep_service; pub mod schema_change; pub mod shuffling_cache; -pub mod snapshot_cache; pub mod state_advance_timer; pub mod sync_committee_rewards; pub mod sync_committee_verification; diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index df718413cc0..fc3f032cdc3 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -4,12 +4,8 @@ use crate::{BeaconChain, BeaconChainError, BeaconChainTypes}; use lazy_static::lazy_static; pub use lighthouse_metrics::*; use slot_clock::SlotClock; -use std::time::Duration; use types::{BeaconState, Epoch, EthSpec, Hash256, Slot}; -/// The maximum time to wait for the snapshot cache lock during a metrics scrape. -const SNAPSHOT_CACHE_TIMEOUT: Duration = Duration::from_millis(100); - // Attestation simulator metrics pub const VALIDATOR_MONITOR_ATTESTATION_SIMULATOR_HEAD_ATTESTER_HIT_TOTAL: &str = "validator_monitor_attestation_simulator_head_attester_hit_total"; @@ -1204,15 +1200,10 @@ pub fn scrape_for_metrics(beacon_chain: &BeaconChain) { let attestation_stats = beacon_chain.op_pool.attestation_stats(); - if let Some(snapshot_cache) = beacon_chain - .snapshot_cache - .try_write_for(SNAPSHOT_CACHE_TIMEOUT) - { - set_gauge( - &BLOCK_PROCESSING_SNAPSHOT_CACHE_SIZE, - snapshot_cache.len() as i64, - ) - } + set_gauge_by_usize( + &BLOCK_PROCESSING_SNAPSHOT_CACHE_SIZE, + beacon_chain.store.state_cache_len(), + ); set_gauge_by_usize( &BEACON_REQRESP_PRE_IMPORT_CACHE_SIZE, diff --git a/beacon_node/beacon_chain/src/shuffling_cache.rs b/beacon_node/beacon_chain/src/shuffling_cache.rs index b3de6f91c92..04d58882639 100644 --- a/beacon_node/beacon_chain/src/shuffling_cache.rs +++ b/beacon_node/beacon_chain/src/shuffling_cache.rs @@ -339,7 +339,7 @@ mod test { .clone(); let committee_b = state.committee_cache(RelativeEpoch::Next).unwrap().clone(); assert!(committee_a != committee_b); - (Arc::new(committee_a), Arc::new(committee_b)) + (committee_a, committee_b) } /// Builds a deterministic but incoherent shuffling ID from a `u64`. diff --git a/beacon_node/beacon_chain/src/state_advance_timer.rs b/beacon_node/beacon_chain/src/state_advance_timer.rs index 39d35f81113..1f928a16e42 100644 --- a/beacon_node/beacon_chain/src/state_advance_timer.rs +++ b/beacon_node/beacon_chain/src/state_advance_timer.rs @@ -15,9 +15,7 @@ //! 2. There's a possibility that the head block is never built upon, causing wasted CPU cycles. use crate::validator_monitor::HISTORIC_EPOCHS as VALIDATOR_MONITOR_HISTORIC_EPOCHS; use crate::{ - beacon_chain::{ATTESTATION_CACHE_LOCK_TIMEOUT, BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT}, - chain_config::FORK_CHOICE_LOOKAHEAD_FACTOR, - snapshot_cache::StateAdvance, + beacon_chain::ATTESTATION_CACHE_LOCK_TIMEOUT, chain_config::FORK_CHOICE_LOOKAHEAD_FACTOR, BeaconChain, BeaconChainError, BeaconChainTypes, }; use slog::{debug, error, warn, Logger}; @@ -27,9 +25,10 @@ use std::sync::{ atomic::{AtomicBool, Ordering}, Arc, }; +use store::KeyValueStore; use task_executor::TaskExecutor; use tokio::time::{sleep, sleep_until, Instant}; -use types::{AttestationShufflingId, EthSpec, Hash256, RelativeEpoch, Slot}; +use types::{AttestationShufflingId, BeaconStateError, EthSpec, Hash256, RelativeEpoch, Slot}; /// If the head slot is more than `MAX_ADVANCE_DISTANCE` from the current slot, then don't perform /// the state advancement. @@ -45,14 +44,13 @@ const MAX_ADVANCE_DISTANCE: u64 = 4; /// impact whilst having 8 epochs without a block is a comfortable grace period. const MAX_FORK_CHOICE_DISTANCE: u64 = 256; -/// Drop any unused block production state cache after this many slots. -const MAX_BLOCK_PRODUCTION_CACHE_DISTANCE: u64 = 4; - #[derive(Debug)] enum Error { BeaconChain(BeaconChainError), // We don't use the inner value directly, but it's used in the Debug impl. HeadMissingFromSnapshotCache(#[allow(dead_code)] Hash256), + BeaconState(#[allow(dead_code)] BeaconStateError), + Store(#[allow(dead_code)] store::Error), MaxDistanceExceeded { current_slot: Slot, head_slot: Slot, @@ -72,6 +70,18 @@ impl From for Error { } } +impl From for Error { + fn from(e: BeaconStateError) -> Self { + Self::BeaconState(e) + } +} + +impl From for Error { + fn from(e: store::Error) -> Self { + Self::Store(e) + } +} + /// Provides a simple thread-safe lock to be used for task co-ordination. Practically equivalent to /// `Mutex<()>`. #[derive(Clone)] @@ -231,7 +241,7 @@ async fn state_advance_timer( // Prepare proposers so that the node can send payload attributes in the case where // it decides to abandon a proposer boost re-org. - let proposer_head = beacon_chain + beacon_chain .prepare_beacon_proposer(current_slot) .await .unwrap_or_else(|e| { @@ -248,56 +258,6 @@ async fn state_advance_timer( // in `ForkChoiceSignalTx`. beacon_chain.task_executor.clone().spawn_blocking( move || { - // If we're proposing, clone the head state preemptively so that it isn't on - // the hot path of proposing. We can delete this once we have tree-states. - if let Some(proposer_head) = proposer_head { - let mut cache = beacon_chain.block_production_state.lock(); - - // Avoid holding two states in memory. It's OK to hold the lock because - // we always lock the block production cache before the snapshot cache - // and we prefer for block production to wait for the block production - // cache if a clone is in-progress. - if cache - .as_ref() - .map_or(false, |(cached_head, _)| *cached_head != proposer_head) - { - drop(cache.take()); - } - if let Some(proposer_state) = beacon_chain - .snapshot_cache - .try_read_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) - .and_then(|snapshot_cache| { - snapshot_cache.get_state_for_block_production(proposer_head) - }) - { - *cache = Some((proposer_head, proposer_state)); - debug!( - log, - "Cloned state ready for block production"; - "head_block_root" => ?proposer_head, - "slot" => next_slot - ); - } else { - warn!( - log, - "Block production state missing from snapshot cache"; - "head_block_root" => ?proposer_head, - "slot" => next_slot - ); - } - } else { - // If we aren't proposing, drop any old block production cache to save - // memory. - let mut cache = beacon_chain.block_production_state.lock(); - if let Some((_, state)) = &*cache { - if state.pre_state.slot() + MAX_BLOCK_PRODUCTION_CACHE_DISTANCE - <= current_slot - { - drop(cache.take()); - } - } - } - // Signal block proposal for the next slot (if it happens to be waiting). if let Some(tx) = &beacon_chain.fork_choice_signal_tx { if let Err(e) = tx.notify_fork_choice_complete(next_slot) { @@ -318,9 +278,9 @@ async fn state_advance_timer( } } -/// Reads the `snapshot_cache` from the `beacon_chain` and attempts to take a clone of the +/// Reads the `state_cache` from the `beacon_chain` and attempts to take a clone of the /// `BeaconState` of the head block. If it obtains this clone, the state will be advanced a single -/// slot then placed back in the `snapshot_cache` to be used for block verification. +/// slot then placed in the `state_cache` to be used for block verification. /// /// See the module-level documentation for rationale. fn advance_head( @@ -345,46 +305,42 @@ fn advance_head( } } - let head_root = beacon_chain.head_beacon_block_root(); + let (head_block_root, head_block_state_root) = { + let snapshot = beacon_chain.head_snapshot(); + (snapshot.beacon_block_root, snapshot.beacon_state_root()) + }; - let (head_slot, head_state_root, mut state) = match beacon_chain - .snapshot_cache - .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) - .ok_or(BeaconChainError::SnapshotCacheLockTimeout)? - .get_for_state_advance(head_root) - { - StateAdvance::AlreadyAdvanced => { + let (head_state_root, mut state) = beacon_chain + .store + .get_advanced_hot_state(head_block_root, current_slot, head_block_state_root)? + .ok_or(Error::HeadMissingFromSnapshotCache(head_block_root))?; + + // Protect against advancing a state more than a single slot. + // + // Advancing more than one slot without storing the intermediate state would corrupt the + // database. Future works might store temporary, intermediate states inside this function. + match state.slot().cmp(&state.latest_block_header().slot) { + std::cmp::Ordering::Equal => (), + std::cmp::Ordering::Greater => { return Err(Error::StateAlreadyAdvanced { - block_root: head_root, - }) + block_root: head_block_root, + }); } - StateAdvance::BlockNotFound => return Err(Error::HeadMissingFromSnapshotCache(head_root)), - StateAdvance::State { - state, - state_root, - block_slot, - } => (block_slot, state_root, *state), - }; + std::cmp::Ordering::Less => { + return Err(Error::BadStateSlot { + _block_slot: state.latest_block_header().slot, + _state_slot: state.slot(), + }); + } + } let initial_slot = state.slot(); let initial_epoch = state.current_epoch(); - let state_root = if state.slot() == head_slot { - Some(head_state_root) - } else { - // Protect against advancing a state more than a single slot. - // - // Advancing more than one slot without storing the intermediate state would corrupt the - // database. Future works might store temporary, intermediate states inside this function. - return Err(Error::BadStateSlot { - _block_slot: head_slot, - _state_slot: state.slot(), - }); - }; - // Advance the state a single slot. - if let Some(summary) = per_slot_processing(&mut state, state_root, &beacon_chain.spec) - .map_err(BeaconChainError::from)? + if let Some(summary) = + per_slot_processing(&mut state, Some(head_state_root), &beacon_chain.spec) + .map_err(BeaconChainError::from)? { // Expose Prometheus metrics. if let Err(e) = summary.observe_metrics() { @@ -418,7 +374,7 @@ fn advance_head( debug!( log, "Advanced head state one slot"; - "head_root" => ?head_root, + "head_block_root" => ?head_block_root, "state_slot" => state.slot(), "current_slot" => current_slot, ); @@ -437,14 +393,14 @@ fn advance_head( if initial_epoch < state.current_epoch() { // Update the proposer cache. // - // We supply the `head_root` as the decision block since the prior `if` statement guarantees + // We supply the `head_block_root` as the decision block since the prior `if` statement guarantees // the head root is the latest block from the prior epoch. beacon_chain .beacon_proposer_cache .lock() .insert( state.current_epoch(), - head_root, + head_block_root, state .get_beacon_proposer_indices(&beacon_chain.spec) .map_err(BeaconChainError::from)?, @@ -453,8 +409,9 @@ fn advance_head( .map_err(BeaconChainError::from)?; // Update the attester cache. - let shuffling_id = AttestationShufflingId::new(head_root, &state, RelativeEpoch::Next) - .map_err(BeaconChainError::from)?; + let shuffling_id = + AttestationShufflingId::new(head_block_root, &state, RelativeEpoch::Next) + .map_err(BeaconChainError::from)?; let committee_cache = state .committee_cache(RelativeEpoch::Next) .map_err(BeaconChainError::from)?; @@ -467,7 +424,7 @@ fn advance_head( debug!( log, "Primed proposer and attester caches"; - "head_root" => ?head_root, + "head_block_root" => ?head_block_root, "next_epoch_shuffling_root" => ?shuffling_id.shuffling_decision_block, "state_epoch" => state.current_epoch(), "current_epoch" => current_slot.epoch(T::EthSpec::slots_per_epoch()), @@ -477,22 +434,13 @@ fn advance_head( // Apply the state to the attester cache, if the cache deems it interesting. beacon_chain .attester_cache - .maybe_cache_state(&state, head_root, &beacon_chain.spec) + .maybe_cache_state(&state, head_block_root, &beacon_chain.spec) .map_err(BeaconChainError::from)?; let final_slot = state.slot(); - // Insert the advanced state back into the snapshot cache. - beacon_chain - .snapshot_cache - .try_write_for(BLOCK_PROCESSING_CACHE_LOCK_TIMEOUT) - .ok_or(BeaconChainError::SnapshotCacheLockTimeout)? - .update_pre_state(head_root, state) - .ok_or(Error::HeadMissingFromSnapshotCache(head_root))?; - // If we have moved into the next slot whilst processing the state then this function is going - // to become ineffective and likely become a hindrance as we're stealing the tree hash cache - // from the snapshot cache (which may force the next block to rebuild a new one). + // to become ineffective. // // If this warning occurs very frequently on well-resourced machines then we should consider // starting it earlier in the slot. Otherwise, it's a good indication that the machine is too @@ -503,7 +451,7 @@ fn advance_head( warn!( log, "State advance too slow"; - "head_root" => %head_root, + "head_block_root" => %head_block_root, "advanced_slot" => final_slot, "current_slot" => current_slot, "starting_slot" => starting_slot, @@ -511,10 +459,25 @@ fn advance_head( ); } + // Write the advanced state to the database with a temporary flag that will be deleted when + // a block is imported on top of this state. We should delete this once we bring in the DB + // changes from tree-states that allow us to prune states without temporary flags. + let advanced_state_root = state.update_tree_hash_cache()?; + let txn_lock = beacon_chain.store.hot_db.begin_rw_transaction(); + let state_already_exists = beacon_chain + .store + .load_hot_state_summary(&advanced_state_root)? + .is_some(); + let temporary = !state_already_exists; + beacon_chain + .store + .put_state_possibly_temporary(&advanced_state_root, &state, temporary)?; + drop(txn_lock); + debug!( log, "Completed state advance"; - "head_root" => ?head_root, + "head_block_root" => ?head_block_root, "advanced_slot" => final_slot, "initial_slot" => initial_slot, ); diff --git a/beacon_node/beacon_chain/src/sync_committee_rewards.rs b/beacon_node/beacon_chain/src/sync_committee_rewards.rs index 2221aa1d5eb..9b35cff9432 100644 --- a/beacon_node/beacon_chain/src/sync_committee_rewards.rs +++ b/beacon_node/beacon_chain/src/sync_committee_rewards.rs @@ -38,9 +38,26 @@ impl BeaconChain { })?; let mut balances = HashMap::::new(); + for &validator_index in &sync_committee_indices { + balances.insert( + validator_index, + *state + .balances() + .get(validator_index) + .ok_or(BeaconChainError::SyncCommitteeRewardsSyncError)?, + ); + } + + let proposer_index = block.proposer_index() as usize; + balances.insert( + proposer_index, + *state + .balances() + .get(proposer_index) + .ok_or(BeaconChainError::SyncCommitteeRewardsSyncError)?, + ); let mut total_proposer_rewards = 0; - let proposer_index = state.get_beacon_proposer_index(block.slot(), spec)?; // Apply rewards to participant balances. Keep track of proposer rewards for (validator_index, participant_bit) in sync_committee_indices @@ -48,15 +65,15 @@ impl BeaconChain { .zip(sync_aggregate.sync_committee_bits.iter()) { let participant_balance = balances - .entry(*validator_index) - .or_insert_with(|| state.balances()[*validator_index]); + .get_mut(validator_index) + .ok_or(BeaconChainError::SyncCommitteeRewardsSyncError)?; if participant_bit { participant_balance.safe_add_assign(participant_reward_value)?; balances - .entry(proposer_index) - .or_insert_with(|| state.balances()[proposer_index]) + .get_mut(&proposer_index) + .ok_or(BeaconChainError::SyncCommitteeRewardsSyncError)? .safe_add_assign(proposer_reward_per_bit)?; total_proposer_rewards.safe_add_assign(proposer_reward_per_bit)?; @@ -67,18 +84,17 @@ impl BeaconChain { Ok(balances .iter() - .filter_map(|(i, new_balance)| { - let reward = if *i != proposer_index { - *new_balance as i64 - state.balances()[*i] as i64 - } else if sync_committee_indices.contains(i) { - *new_balance as i64 - - state.balances()[*i] as i64 - - total_proposer_rewards as i64 + .filter_map(|(&i, &new_balance)| { + let initial_balance = *state.balances().get(i)? as i64; + let reward = if i != proposer_index { + new_balance as i64 - initial_balance + } else if sync_committee_indices.contains(&i) { + new_balance as i64 - initial_balance - total_proposer_rewards as i64 } else { return None; }; Some(SyncCommitteeReward { - validator_index: *i as u64, + validator_index: i as u64, reward, }) }) diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index debc4881a60..d3eeff8a7d9 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -46,10 +46,7 @@ use slog_async::Async; use slog_term::{FullFormat, TermDecorator}; use slot_clock::{SlotClock, TestingSlotClock}; use state_processing::per_block_processing::compute_timestamp_at_slot; -use state_processing::{ - state_advance::{complete_state_advance, partial_state_advance}, - StateProcessingStrategy, -}; +use state_processing::state_advance::complete_state_advance; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::fmt; @@ -756,10 +753,7 @@ where pub fn get_current_state_and_root(&self) -> (BeaconState, Hash256) { let head = self.chain.head_snapshot(); let state_root = head.beacon_state_root(); - ( - head.beacon_state.clone_with_only_committee_caches(), - state_root, - ) + (head.beacon_state.clone(), state_root) } pub fn head_slot(&self) -> Slot { @@ -802,8 +796,9 @@ where pub fn get_hot_state(&self, state_hash: BeaconStateHash) -> Option> { self.chain .store - .load_hot_state(&state_hash.into(), StateProcessingStrategy::Accurate) + .load_hot_state(&state_hash.into()) .unwrap() + .map(|(state, _)| state) } pub fn get_cold_state(&self, state_hash: BeaconStateHash) -> Option> { @@ -1017,9 +1012,7 @@ where return Err(BeaconChainError::CannotAttestToFutureState); } else if state.current_epoch() < epoch { let mut_state = state.to_mut(); - // Only perform a "partial" state advance since we do not require the state roots to be - // accurate. - partial_state_advance( + complete_state_advance( mut_state, Some(state_root), epoch.start_slot(E::slots_per_epoch()), diff --git a/beacon_node/beacon_chain/src/validator_pubkey_cache.rs b/beacon_node/beacon_chain/src/validator_pubkey_cache.rs index 2cf0c326158..e1b50706286 100644 --- a/beacon_node/beacon_chain/src/validator_pubkey_cache.rs +++ b/beacon_node/beacon_chain/src/validator_pubkey_cache.rs @@ -81,8 +81,9 @@ impl ValidatorPubkeyCache { ) -> Result>, BeaconChainError> { if state.validators().len() > self.pubkeys.len() { self.import( - state.validators()[self.pubkeys.len()..] - .iter() + state + .validators() + .iter_from(self.pubkeys.len())? .map(|v| v.pubkey), ) } else { diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 4d37557f0d1..98a112daffe 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -15,8 +15,7 @@ use slasher::{Config as SlasherConfig, Slasher}; use state_processing::{ common::get_indexed_attestation, per_block_processing::{per_block_processing, BlockSignatureStrategy}, - per_slot_processing, BlockProcessingError, ConsensusContext, StateProcessingStrategy, - VerifyBlockRoot, + per_slot_processing, BlockProcessingError, ConsensusContext, VerifyBlockRoot, }; use std::marker::PhantomData; use std::sync::Arc; @@ -1309,7 +1308,6 @@ async fn add_base_block_to_altair_chain() { &mut state, &base_block, BlockSignatureStrategy::NoVerification, - StateProcessingStrategy::Accurate, VerifyBlockRoot::True, &mut ctxt, &harness.chain.spec, @@ -1445,7 +1443,6 @@ async fn add_altair_block_to_base_chain() { &mut state, &altair_block, BlockSignatureStrategy::NoVerification, - StateProcessingStrategy::Accurate, VerifyBlockRoot::True, &mut ctxt, &harness.chain.spec, diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index f1262596f70..0ef348319af 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -223,7 +223,7 @@ impl InvalidPayloadRig { let mock_execution_layer = self.harness.mock_execution_layer.as_ref().unwrap(); let head = self.harness.chain.head_snapshot(); - let state = head.beacon_state.clone_with_only_committee_caches(); + let state = head.beacon_state.clone(); let slot = slot_override.unwrap_or(state.slot() + 1); let ((block, blobs), post_state) = self.harness.make_block(state, slot).await; let block_root = block.canonical_root(); @@ -2048,7 +2048,7 @@ async fn weights_after_resetting_optimistic_status() { .fork_choice_read_lock() .get_block_weight(&head.head_block_root()) .unwrap(), - head.snapshot.beacon_state.validators()[0].effective_balance, + head.snapshot.beacon_state.validators().get(0).unwrap().effective_balance, "proposer boost should be removed from the head block and the vote of a single validator applied" ); diff --git a/beacon_node/beacon_chain/tests/rewards.rs b/beacon_node/beacon_chain/tests/rewards.rs index a78463ef5d7..1c80525223a 100644 --- a/beacon_node/beacon_chain/tests/rewards.rs +++ b/beacon_node/beacon_chain/tests/rewards.rs @@ -105,8 +105,8 @@ async fn test_sync_committee_rewards() { .get_validator_index(&validator.pubkey) .unwrap() .unwrap(); - let pre_state_balance = parent_state.balances()[validator_index]; - let post_state_balance = state.balances()[validator_index]; + let pre_state_balance = *parent_state.balances().get(validator_index).unwrap(); + let post_state_balance = *state.balances().get(validator_index).unwrap(); let sync_committee_reward = rewards.get(&(validator_index as u64)).unwrap_or(&0); if validator_index == proposer_index { @@ -141,7 +141,7 @@ async fn test_verify_attestation_rewards_base() { ) .await; - let initial_balances: Vec = harness.get_current_state().balances().clone().into(); + let initial_balances: Vec = harness.get_current_state().balances().to_vec(); // extend slots to beginning of epoch N + 2 harness.extend_slots(E::slots_per_epoch() as usize).await; @@ -163,7 +163,7 @@ async fn test_verify_attestation_rewards_base() { let expected_balances = apply_attestation_rewards(&initial_balances, total_rewards); // verify expected balances against actual balances - let balances: Vec = harness.get_current_state().balances().clone().into(); + let balances: Vec = harness.get_current_state().balances().to_vec(); assert_eq!(expected_balances, balances); } @@ -185,7 +185,7 @@ async fn test_verify_attestation_rewards_base_inactivity_leak() { AttestationStrategy::SomeValidators(half_validators.clone()), ) .await; - let initial_balances: Vec = harness.get_current_state().balances().clone().into(); + let initial_balances: Vec = harness.get_current_state().balances().to_vec(); // extend slots to beginning of epoch N + 2 harness.advance_slot(); @@ -215,7 +215,7 @@ async fn test_verify_attestation_rewards_base_inactivity_leak() { let expected_balances = apply_attestation_rewards(&initial_balances, total_rewards); // verify expected balances against actual balances - let balances: Vec = harness.get_current_state().balances().clone().into(); + let balances: Vec = harness.get_current_state().balances().to_vec(); assert_eq!(expected_balances, balances); } @@ -241,7 +241,7 @@ async fn test_verify_attestation_rewards_base_inactivity_leak_justification_epoc // advance to create first justification epoch and get initial balances harness.extend_slots(E::slots_per_epoch() as usize).await; target_epoch += 1; - let initial_balances: Vec = harness.get_current_state().balances().clone().into(); + let initial_balances: Vec = harness.get_current_state().balances().to_vec(); //assert previous_justified_checkpoint matches 0 as we were in inactivity leak from beginning assert_eq!( @@ -284,7 +284,7 @@ async fn test_verify_attestation_rewards_base_inactivity_leak_justification_epoc let expected_balances = apply_attestation_rewards(&initial_balances, total_rewards); // verify expected balances against actual balances - let balances: Vec = harness.get_current_state().balances().clone().into(); + let balances: Vec = harness.get_current_state().balances().to_vec(); assert_eq!(expected_balances, balances); } @@ -298,7 +298,7 @@ async fn test_verify_attestation_rewards_altair() { harness .extend_slots((E::slots_per_epoch() * (target_epoch + 1)) as usize) .await; - let initial_balances: Vec = harness.get_current_state().balances().clone().into(); + let initial_balances: Vec = harness.get_current_state().balances().to_vec(); // advance until epoch N + 2 and build proposal rewards map let mut proposal_rewards_map: HashMap = HashMap::new(); @@ -364,7 +364,7 @@ async fn test_verify_attestation_rewards_altair() { apply_sync_committee_rewards(&sync_committee_rewards_map, expected_balances); // verify expected balances against actual balances - let balances: Vec = harness.get_current_state().balances().clone().into(); + let balances: Vec = harness.get_current_state().balances().to_vec(); assert_eq!(expected_balances, balances); } @@ -386,7 +386,7 @@ async fn test_verify_attestation_rewards_altair_inactivity_leak() { half_validators.clone(), ) .await; - let initial_balances: Vec = harness.get_current_state().balances().clone().into(); + let initial_balances: Vec = harness.get_current_state().balances().to_vec(); // advance until epoch N + 2 and build proposal rewards map let mut proposal_rewards_map: HashMap = HashMap::new(); @@ -458,7 +458,7 @@ async fn test_verify_attestation_rewards_altair_inactivity_leak() { apply_sync_committee_rewards(&sync_committee_rewards_map, expected_balances); // verify expected balances against actual balances - let balances: Vec = harness.get_current_state().balances().clone().into(); + let balances: Vec = harness.get_current_state().balances().to_vec(); assert_eq!(expected_balances, balances); } @@ -492,7 +492,7 @@ async fn test_verify_attestation_rewards_altair_inactivity_leak_justification_ep // advance for first justification epoch and get balances harness.extend_slots(E::slots_per_epoch() as usize).await; target_epoch += 1; - let initial_balances: Vec = harness.get_current_state().balances().clone().into(); + let initial_balances: Vec = harness.get_current_state().balances().to_vec(); // advance until epoch N + 2 and build proposal rewards map let mut proposal_rewards_map: HashMap = HashMap::new(); @@ -568,7 +568,7 @@ async fn test_verify_attestation_rewards_altair_inactivity_leak_justification_ep apply_sync_committee_rewards(&sync_committee_rewards_map, expected_balances); // verify expected balances against actual balances - let balances: Vec = harness.get_current_state().balances().clone().into(); + let balances: Vec = harness.get_current_state().balances().to_vec(); assert_eq!(expected_balances, balances); } diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index 51d62cf8a19..ba8a6bf7016 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -719,52 +719,6 @@ async fn forwards_iter_block_and_state_roots_until() { test_range(Slot::new(0), head_state.slot()); } -#[tokio::test] -async fn block_replay_with_inaccurate_state_roots() { - let num_blocks_produced = E::slots_per_epoch() * 3 + 31; - let db_path = tempdir().unwrap(); - let store = get_store(&db_path); - let harness = get_harness(store.clone(), LOW_VALIDATOR_COUNT); - let chain = &harness.chain; - - harness - .extend_chain( - num_blocks_produced as usize, - BlockStrategy::OnCanonicalHead, - AttestationStrategy::AllValidators, - ) - .await; - - // Slot must not be 0 mod 32 or else no blocks will be replayed. - let (mut head_state, head_state_root) = harness.get_current_state_and_root(); - let head_block_root = harness.head_block_root(); - assert_ne!(head_state.slot() % 32, 0); - - let (_, mut fast_head_state) = store - .get_inconsistent_state_for_attestation_verification_only( - &head_block_root, - head_state.slot(), - head_state_root, - ) - .unwrap() - .unwrap(); - assert_eq!(head_state.validators(), fast_head_state.validators()); - - head_state.build_all_committee_caches(&chain.spec).unwrap(); - fast_head_state - .build_all_committee_caches(&chain.spec) - .unwrap(); - - assert_eq!( - head_state - .get_cached_active_validator_indices(RelativeEpoch::Current) - .unwrap(), - fast_head_state - .get_cached_active_validator_indices(RelativeEpoch::Current) - .unwrap() - ); -} - #[tokio::test] async fn block_replayer_hooks() { let db_path = tempdir().unwrap(); @@ -795,7 +749,7 @@ async fn block_replayer_hooks() { let mut post_block_slots = vec![]; let mut replay_state = BlockReplayer::::new(state, &chain.spec) - .pre_slot_hook(Box::new(|state| { + .pre_slot_hook(Box::new(|_, state| { pre_slots.push(state.slot()); Ok(()) })) @@ -834,6 +788,8 @@ async fn block_replayer_hooks() { assert_eq!(post_block_slots, block_slots); // States match. + end_state.apply_pending_mutations().unwrap(); + replay_state.apply_pending_mutations().unwrap(); end_state.drop_all_caches().unwrap(); replay_state.drop_all_caches().unwrap(); assert_eq!(end_state, replay_state); @@ -1219,9 +1175,17 @@ fn check_shuffling_compatible( |committee_cache, _| { let state_cache = head_state.committee_cache(RelativeEpoch::Current).unwrap(); if current_epoch_shuffling_is_compatible { - assert_eq!(committee_cache, state_cache, "block at slot {slot}"); + assert_eq!( + committee_cache, + state_cache.as_ref(), + "block at slot {slot}" + ); } else { - assert_ne!(committee_cache, state_cache, "block at slot {slot}"); + assert_ne!( + committee_cache, + state_cache.as_ref(), + "block at slot {slot}" + ); } Ok(()) }, @@ -1251,9 +1215,9 @@ fn check_shuffling_compatible( |committee_cache, _| { let state_cache = head_state.committee_cache(RelativeEpoch::Previous).unwrap(); if previous_epoch_shuffling_is_compatible { - assert_eq!(committee_cache, state_cache); + assert_eq!(committee_cache, state_cache.as_ref()); } else { - assert_ne!(committee_cache, state_cache); + assert_ne!(committee_cache, state_cache.as_ref()); } Ok(()) }, @@ -3605,16 +3569,16 @@ fn check_split_slot(harness: &TestHarness, store: Arc, L /// Check that all the states in a chain dump have the correct tree hash. fn check_chain_dump(harness: &TestHarness, expected_len: u64) { - let chain_dump = harness.chain.chain_dump().unwrap(); + let mut chain_dump = harness.chain.chain_dump().unwrap(); let split_slot = harness.chain.store.get_split_slot(); assert_eq!(chain_dump.len() as u64, expected_len); - for checkpoint in &chain_dump { + for checkpoint in &mut chain_dump { // Check that the tree hash of the stored state is as expected assert_eq!( checkpoint.beacon_state_root(), - checkpoint.beacon_state.tree_hash_root(), + checkpoint.beacon_state.update_tree_hash_cache().unwrap(), "tree hash of stored state is incorrect" ); diff --git a/beacon_node/genesis/src/interop.rs b/beacon_node/genesis/src/interop.rs index b4753e92f1f..4c78b8efd8f 100644 --- a/beacon_node/genesis/src/interop.rs +++ b/beacon_node/genesis/src/interop.rs @@ -178,13 +178,14 @@ mod test { } for v in state.validators() { - let creds = v.withdrawal_credentials.as_bytes(); + let creds = v.withdrawal_credentials; assert_eq!( - creds[0], spec.bls_withdrawal_prefix_byte, + creds.as_bytes()[0], + spec.bls_withdrawal_prefix_byte, "first byte of withdrawal creds should be bls prefix" ); assert_eq!( - &creds[1..], + &creds.as_bytes()[1..], &hash(&v.pubkey.as_ssz_bytes())[1..], "rest of withdrawal creds should be pubkey hash" ) @@ -240,7 +241,8 @@ mod test { } for (index, v) in state.validators().iter().enumerate() { - let creds = v.withdrawal_credentials.as_bytes(); + let withdrawal_credientials = v.withdrawal_credentials; + let creds = withdrawal_credientials.as_bytes(); if index % 2 == 0 { assert_eq!( creds[0], spec.bls_withdrawal_prefix_byte, diff --git a/beacon_node/http_api/src/attester_duties.rs b/beacon_node/http_api/src/attester_duties.rs index f3242a2b374..6c7dc3348c1 100644 --- a/beacon_node/http_api/src/attester_duties.rs +++ b/beacon_node/http_api/src/attester_duties.rs @@ -5,9 +5,7 @@ use beacon_chain::{BeaconChain, BeaconChainError, BeaconChainTypes}; use eth2::types::{self as api_types}; use slot_clock::SlotClock; use state_processing::state_advance::partial_state_advance; -use types::{ - AttestationDuty, BeaconState, ChainSpec, CloneConfig, Epoch, EthSpec, Hash256, RelativeEpoch, -}; +use types::{AttestationDuty, BeaconState, ChainSpec, Epoch, EthSpec, Hash256, RelativeEpoch}; /// The struct that is returned to the requesting HTTP client. type ApiDuties = api_types::DutiesResponse>; @@ -90,8 +88,7 @@ fn compute_historic_attester_duties( if head.beacon_state.current_epoch() <= request_epoch { Some(( head.beacon_state_root(), - head.beacon_state - .clone_with(CloneConfig::committee_caches_only()), + head.beacon_state.clone(), execution_status.is_optimistic_or_invalid(), )) } else { diff --git a/beacon_node/http_api/src/block_packing_efficiency.rs b/beacon_node/http_api/src/block_packing_efficiency.rs index d78f1f7c66e..f105fdf0a7d 100644 --- a/beacon_node/http_api/src/block_packing_efficiency.rs +++ b/beacon_node/http_api/src/block_packing_efficiency.rs @@ -279,7 +279,7 @@ pub fn get_block_packing_efficiency( )); let pre_slot_hook = - |state: &mut BeaconState| -> Result<(), PackingEfficiencyError> { + |_, state: &mut BeaconState| -> Result<(), PackingEfficiencyError> { // Add attestations to `available_attestations`. handler.lock().add_attestations(state.slot())?; Ok(()) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index cc117c3fb92..46d3ad7569b 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -61,7 +61,6 @@ use slog::{crit, debug, error, info, warn, Logger}; use slot_clock::SlotClock; use ssz::Encode; pub use state_id::StateId; -use std::borrow::Cow; use std::future::Future; use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::path::PathBuf; @@ -864,10 +863,10 @@ pub fn serve( None }; - let committee_cache = if let Some(ref shuffling) = + let committee_cache = if let Some(shuffling) = maybe_cached_shuffling { - Cow::Borrowed(&**shuffling) + shuffling } else { let possibly_built_cache = match RelativeEpoch::from_epoch(current_epoch, epoch) { @@ -876,16 +875,13 @@ pub fn serve( relative_epoch, ) => { - state - .committee_cache(relative_epoch) - .map(Cow::Borrowed) + state.committee_cache(relative_epoch).cloned() } _ => CommitteeCache::initialized( state, epoch, &chain.spec, - ) - .map(Cow::Owned), + ), } .map_err(|e| { match e { @@ -933,7 +929,7 @@ pub fn serve( { cache_write.insert_committee_cache( shuffling_id, - &*possibly_built_cache, + &possibly_built_cache, ); } } diff --git a/beacon_node/http_api/src/proposer_duties.rs b/beacon_node/http_api/src/proposer_duties.rs index c31dd9b1faa..ab8952976c8 100644 --- a/beacon_node/http_api/src/proposer_duties.rs +++ b/beacon_node/http_api/src/proposer_duties.rs @@ -10,7 +10,7 @@ use safe_arith::SafeArith; use slog::{debug, Logger}; use slot_clock::SlotClock; use std::cmp::Ordering; -use types::{CloneConfig, Epoch, EthSpec, Hash256, Slot}; +use types::{Epoch, EthSpec, Hash256, Slot}; /// The struct that is returned to the requesting HTTP client. type ApiDuties = api_types::DutiesResponse>; @@ -192,8 +192,7 @@ fn compute_historic_proposer_duties( if head.beacon_state.current_epoch() <= epoch { Some(( head.beacon_state_root(), - head.beacon_state - .clone_with(CloneConfig::committee_caches_only()), + head.beacon_state.clone(), execution_status.is_optimistic_or_invalid(), )) } else { diff --git a/beacon_node/http_api/src/state_id.rs b/beacon_node/http_api/src/state_id.rs index 1a76333e2d4..fdc99fa954e 100644 --- a/beacon_node/http_api/src/state_id.rs +++ b/beacon_node/http_api/src/state_id.rs @@ -178,10 +178,7 @@ impl StateId { .head_and_execution_status() .map_err(warp_utils::reject::beacon_chain_error)?; return Ok(( - cached_head - .snapshot - .beacon_state - .clone_with_only_committee_caches(), + cached_head.snapshot.beacon_state.clone(), execution_status.is_optimistic_or_invalid(), false, )); diff --git a/beacon_node/http_api/tests/fork_tests.rs b/beacon_node/http_api/tests/fork_tests.rs index 74b26475639..ad32ff1d579 100644 --- a/beacon_node/http_api/tests/fork_tests.rs +++ b/beacon_node/http_api/tests/fork_tests.rs @@ -128,17 +128,18 @@ async fn attestations_across_fork_with_skip_slots() { let all_validators = harness.get_all_validators(); let fork_slot = fork_epoch.start_slot(E::slots_per_epoch()); - let fork_state = harness + let mut fork_state = harness .chain .state_at_slot(fork_slot, StateSkipConfig::WithStateRoots) .unwrap(); + let fork_state_root = fork_state.update_tree_hash_cache().unwrap(); harness.set_current_slot(fork_slot); let attestations = harness.make_attestations( &all_validators, &fork_state, - fork_state.canonical_root(), + fork_state_root, (*fork_state.get_block_root(fork_slot - 1).unwrap()).into(), fork_slot, ); diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index e4580e4ffdb..d44b9a688ce 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -806,7 +806,7 @@ impl ApiTester { let state_opt = state_id.state(&self.chain).ok(); let validators: Vec = match state_opt.as_ref() { Some((state, _execution_optimistic, _finalized)) => { - state.validators().clone().into() + state.validators().clone().to_vec() } None => vec![], }; @@ -822,7 +822,7 @@ impl ApiTester { ValidatorId::PublicKey( validators .get(i as usize) - .map_or(PublicKeyBytes::empty(), |val| val.pubkey.clone()), + .map_or(PublicKeyBytes::empty(), |val| val.pubkey), ) }) .collect::>(); @@ -865,7 +865,7 @@ impl ApiTester { if i < state.balances().len() as u64 { validators.push(ValidatorBalanceData { index: i as u64, - balance: state.balances()[i as usize], + balance: *state.balances().get(i as usize).unwrap(), }); } } @@ -892,7 +892,7 @@ impl ApiTester { .ok() .map(|(state, _execution_optimistic, _finalized)| state); let validators: Vec = match state_opt.as_ref() { - Some(state) => state.validators().clone().into(), + Some(state) => state.validators().to_vec(), None => vec![], }; let validator_index_ids = validator_indices @@ -907,7 +907,7 @@ impl ApiTester { ValidatorId::PublicKey( validators .get(i as usize) - .map_or(PublicKeyBytes::empty(), |val| val.pubkey.clone()), + .map_or(PublicKeyBytes::empty(), |val| val.pubkey), ) }) .collect::>(); @@ -955,7 +955,7 @@ impl ApiTester { if i >= state.validators().len() as u64 { continue; } - let validator = state.validators()[i as usize].clone(); + let validator = state.validators().get(i as usize).unwrap().clone(); let status = ValidatorStatus::from_validator( &validator, epoch, @@ -967,7 +967,7 @@ impl ApiTester { { validators.push(ValidatorData { index: i as u64, - balance: state.balances()[i as usize], + balance: *state.balances().get(i as usize).unwrap(), status, validator, }); @@ -995,13 +995,13 @@ impl ApiTester { .ok() .map(|(state, _execution_optimistic, _finalized)| state); let validators = match state_opt.as_ref() { - Some(state) => state.validators().clone().into(), + Some(state) => state.validators().to_vec(), None => vec![], }; for (i, validator) in validators.into_iter().enumerate() { let validator_ids = &[ - ValidatorId::PublicKey(validator.pubkey.clone()), + ValidatorId::PublicKey(validator.pubkey), ValidatorId::Index(i as u64), ]; @@ -1025,7 +1025,7 @@ impl ApiTester { ValidatorData { index: i as u64, - balance: state.balances()[i], + balance: *state.balances().get(i).unwrap(), status: ValidatorStatus::from_validator( &validator, epoch, @@ -2360,7 +2360,7 @@ impl ApiTester { .unwrap() { let expected = AttesterData { - pubkey: state.validators()[i as usize].pubkey.clone().into(), + pubkey: state.validators().get(i as usize).unwrap().pubkey, validator_index: i, committees_at_slot: duty.committees_at_slot, committee_index: duty.index, @@ -2465,7 +2465,7 @@ impl ApiTester { let index = state .get_beacon_proposer_index(slot, &self.chain.spec) .unwrap(); - let pubkey = state.validators()[index].pubkey.clone().into(); + let pubkey = state.validators().get(index).unwrap().pubkey; ProposerData { pubkey, diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index 7cd40a4bb60..6e744ccf62a 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -1272,7 +1272,7 @@ mod release_tests { // Each validator will have a multiple of 1_000_000_000 wei. // Safe from overflow unless there are about 18B validators (2^64 / 1_000_000_000). for i in 0..state.validators().len() { - state.validators_mut()[i].effective_balance = 1_000_000_000 * i as u64; + state.validators_mut().get_mut(i).unwrap().effective_balance = 1_000_000_000 * i as u64; } let num_validators = num_committees @@ -1530,9 +1530,9 @@ mod release_tests { let spec = &harness.spec; let mut state = harness.get_current_state(); let op_pool = OperationPool::::new(); - state.validators_mut()[1].effective_balance = 17_000_000_000; - state.validators_mut()[2].effective_balance = 17_000_000_000; - state.validators_mut()[3].effective_balance = 17_000_000_000; + state.validators_mut().get_mut(1).unwrap().effective_balance = 17_000_000_000; + state.validators_mut().get_mut(2).unwrap().effective_balance = 17_000_000_000; + state.validators_mut().get_mut(3).unwrap().effective_balance = 17_000_000_000; let slashing_1 = harness.make_attester_slashing(vec![1, 2, 3]); let slashing_2 = harness.make_attester_slashing(vec![4, 5, 6]); diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index fd2cf473cb3..d31b576989c 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -179,9 +179,6 @@ pub fn get_config( if let Some(cache_size) = clap_utils::parse_optional(cli_args, "shuffling-cache-size")? { client_config.chain.shuffling_cache_size = cache_size; } - if let Some(cache_size) = clap_utils::parse_optional(cli_args, "state-cache-size")? { - client_config.chain.snapshot_cache_size = cache_size; - } /* * Prometheus metrics HTTP server @@ -408,6 +405,10 @@ pub fn get_config( .map_err(|_| "block-cache-size is not a valid integer".to_string())?; } + if let Some(cache_size) = clap_utils::parse_optional(cli_args, "state-cache-size")? { + client_config.store.state_cache_size = cache_size; + } + if let Some(historic_state_cache_size) = cli_args.value_of("historic-state-cache-size") { client_config.store.historic_state_cache_size = historic_state_cache_size .parse() diff --git a/beacon_node/store/Cargo.toml b/beacon_node/store/Cargo.toml index 7bf1ef76bef..b7822670078 100644 --- a/beacon_node/store/Cargo.toml +++ b/beacon_node/store/Cargo.toml @@ -25,3 +25,7 @@ lru = { workspace = true } sloggers = { workspace = true } directory = { workspace = true } strum = { workspace = true } +safe_arith = { workspace = true } +bls = { workspace = true } +smallvec = { workspace = true } +logging = { workspace = true } diff --git a/beacon_node/store/src/chunked_vector.rs b/beacon_node/store/src/chunked_vector.rs index a0c50e5a2b5..4450989d590 100644 --- a/beacon_node/store/src/chunked_vector.rs +++ b/beacon_node/store/src/chunked_vector.rs @@ -1,6 +1,6 @@ //! Space-efficient storage for `BeaconState` vector fields. //! -//! This module provides logic for splitting the `FixedVector` fields of a `BeaconState` into +//! This module provides logic for splitting the `Vector` fields of a `BeaconState` into //! chunks, and storing those chunks in contiguous ranges in the on-disk database. The motiviation //! for doing this is avoiding massive duplication in every on-disk state. For example, rather than //! storing the whole `historical_roots` vector, which is updated once every couple of thousand @@ -60,12 +60,13 @@ fn genesis_value_key() -> [u8; 8] { /// type-level. We require their value-level witnesses to be `Copy` so that we can avoid the /// turbofish when calling functions like `store_updated_vector`. pub trait Field: Copy { - /// The type of value stored in this field: the `T` from `FixedVector`. + /// The type of value stored in this field: the `T` from `Vector`. /// /// The `Default` impl will be used to fill extra vector entries. - type Value: Decode + Encode + Default + Clone + PartialEq + std::fmt::Debug; + type Value: Default + std::fmt::Debug + milhouse::Value; + // Decode + Encode + Default + Clone + PartialEq + std::fmt::Debug - /// The length of this field: the `N` from `FixedVector`. + /// The length of this field: the `N` from `Vector`. type Length: Unsigned; /// The database column where the integer-indexed chunks for this field should be stored. @@ -273,10 +274,10 @@ pub trait Field: Copy { } } -/// Marker trait for fixed-length fields (`FixedVector`). +/// Marker trait for fixed-length fields (`Vector`). pub trait FixedLengthField: Field {} -/// Marker trait for variable-length fields (`VariableList`). +/// Marker trait for variable-length fields (`List`). pub trait VariableLengthField: Field {} /// Macro to implement the `Field` trait on a new unit struct type. @@ -331,7 +332,7 @@ field!( activation_slot: Some(Slot::new(0)), deactivation_slot: None }, - |state: &BeaconState<_>, index, _| safe_modulo_index(state.block_roots(), index) + |state: &BeaconState<_>, index, _| safe_modulo_vector_index(state.block_roots(), index) ); field!( @@ -345,7 +346,7 @@ field!( activation_slot: Some(Slot::new(0)), deactivation_slot: None, }, - |state: &BeaconState<_>, index, _| safe_modulo_index(state.state_roots(), index) + |state: &BeaconState<_>, index, _| safe_modulo_vector_index(state.state_roots(), index) ); field!( @@ -361,7 +362,7 @@ field!( .capella_fork_epoch .map(|fork_epoch| fork_epoch.start_slot(E::slots_per_epoch())), }, - |state: &BeaconState<_>, index, _| safe_modulo_index(state.historical_roots(), index) + |state: &BeaconState<_>, index, _| safe_modulo_list_index(state.historical_roots(), index) ); field!( @@ -371,7 +372,7 @@ field!( E::EpochsPerHistoricalVector, DBColumn::BeaconRandaoMixes, |_| OncePerEpoch { lag: 1 }, - |state: &BeaconState<_>, index, _| safe_modulo_index(state.randao_mixes(), index) + |state: &BeaconState<_>, index, _| safe_modulo_vector_index(state.randao_mixes(), index) ); field!( @@ -387,7 +388,7 @@ field!( .map(|fork_epoch| fork_epoch.start_slot(E::slots_per_epoch())), deactivation_slot: None, }, - |state: &BeaconState<_>, index, _| safe_modulo_index( + |state: &BeaconState<_>, index, _| safe_modulo_list_index( state .historical_summaries() .map_err(|_| ChunkError::InvalidFork)?, @@ -565,7 +566,7 @@ pub fn load_vector_from_db, E: EthSpec, S: KeyValueStore< store: &S, slot: Slot, spec: &ChainSpec, -) -> Result, Error> { +) -> Result, Error> { // Do a range query let chunk_size = F::chunk_size(); let (start_vindex, end_vindex) = F::start_and_end_vindex(slot, spec); @@ -589,7 +590,7 @@ pub fn load_vector_from_db, E: EthSpec, S: KeyValueStore< default, )?; - Ok(result.into()) + Ok(Vector::new(result).map_err(ChunkError::Milhouse)?) } /// The historical roots are stored in vector chunks, despite not actually being a vector. @@ -597,7 +598,7 @@ pub fn load_variable_list_from_db, E: EthSpec, S: KeyV store: &S, slot: Slot, spec: &ChainSpec, -) -> Result, Error> { +) -> Result, Error> { let chunk_size = F::chunk_size(); let (start_vindex, end_vindex) = F::start_and_end_vindex(slot, spec); let start_cindex = start_vindex / chunk_size; @@ -617,15 +618,35 @@ pub fn load_variable_list_from_db, E: EthSpec, S: KeyV } } - Ok(result.into()) + Ok(List::new(result).map_err(ChunkError::Milhouse)?) } -/// Index into a field of the state, avoiding out of bounds and division by 0. -fn safe_modulo_index(values: &[T], index: u64) -> Result { +/// Index into a `List` field of the state, avoiding out of bounds and division by 0. +fn safe_modulo_list_index( + values: &List, + index: u64, +) -> Result { + if values.is_empty() { + Err(ChunkError::ZeroLengthList) + } else { + values + .get(index as usize % values.len()) + .copied() + .ok_or(ChunkError::IndexOutOfBounds { index }) + } +} + +fn safe_modulo_vector_index( + values: &Vector, + index: u64, +) -> Result { if values.is_empty() { Err(ChunkError::ZeroLengthVector) } else { - Ok(values[index as usize % values.len()]) + values + .get(index as usize % values.len()) + .copied() + .ok_or(ChunkError::IndexOutOfBounds { index }) } } @@ -712,6 +733,10 @@ where #[derive(Debug, PartialEq)] pub enum ChunkError { ZeroLengthVector, + ZeroLengthList, + IndexOutOfBounds { + index: u64, + }, InvalidSize { chunk_index: usize, expected: usize, @@ -744,6 +769,13 @@ pub enum ChunkError { length: usize, }, InvalidFork, + Milhouse(milhouse::Error), +} + +impl From for ChunkError { + fn from(e: milhouse::Error) -> ChunkError { + Self::Milhouse(e) + } } #[cfg(test)] diff --git a/beacon_node/store/src/config.rs b/beacon_node/store/src/config.rs index 681d424e282..d43999d8220 100644 --- a/beacon_node/store/src/config.rs +++ b/beacon_node/store/src/config.rs @@ -9,6 +9,7 @@ use types::{EthSpec, MinimalEthSpec}; pub const PREV_DEFAULT_SLOTS_PER_RESTORE_POINT: u64 = 2048; pub const DEFAULT_SLOTS_PER_RESTORE_POINT: u64 = 8192; pub const DEFAULT_BLOCK_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(5); +pub const DEFAULT_STATE_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(128); pub const DEFAULT_HISTORIC_STATE_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(1); pub const DEFAULT_EPOCHS_PER_BLOB_PRUNE: u64 = 1; pub const DEFAULT_BLOB_PUNE_MARGIN_EPOCHS: u64 = 0; @@ -22,6 +23,8 @@ pub struct StoreConfig { pub slots_per_restore_point_set_explicitly: bool, /// Maximum number of blocks to store in the in-memory block cache. pub block_cache_size: NonZeroUsize, + /// Maximum number of states to store in the in-memory state cache. + pub state_cache_size: NonZeroUsize, /// Maximum number of states from freezer database to store in the in-memory state cache. pub historic_state_cache_size: NonZeroUsize, /// Whether to compact the database on initialization. @@ -57,6 +60,7 @@ impl Default for StoreConfig { slots_per_restore_point: MinimalEthSpec::slots_per_historical_root() as u64, slots_per_restore_point_set_explicitly: false, block_cache_size: DEFAULT_BLOCK_CACHE_SIZE, + state_cache_size: DEFAULT_STATE_CACHE_SIZE, historic_state_cache_size: DEFAULT_HISTORIC_STATE_CACHE_SIZE, compact_on_init: false, compact_on_prune: true, diff --git a/beacon_node/store/src/errors.rs b/beacon_node/store/src/errors.rs index 96e02b80ff8..91e6a920ba3 100644 --- a/beacon_node/store/src/errors.rs +++ b/beacon_node/store/src/errors.rs @@ -3,7 +3,7 @@ use crate::config::StoreConfigError; use crate::hot_cold_store::HotColdDBError; use ssz::DecodeError; use state_processing::BlockReplayError; -use types::{BeaconStateError, Hash256, InconsistentFork, Slot}; +use types::{BeaconStateError, EpochCacheError, Hash256, InconsistentFork, Slot}; pub type Result = std::result::Result; @@ -49,6 +49,14 @@ pub enum Error { InvalidBytes, UnableToDowngrade, InconsistentFork(InconsistentFork), + CacheBuildError(EpochCacheError), + RandaoMixOutOfBounds, + FinalizedStateDecreasingSlot, + FinalizedStateUnaligned, + StateForCacheHasPendingUpdates { + state_root: Hash256, + slot: Slot, + }, } pub trait HandleUnavailable { @@ -113,6 +121,12 @@ impl From for Error { } } +impl From for Error { + fn from(e: EpochCacheError) -> Error { + Error::CacheBuildError(e) + } +} + #[derive(Debug)] pub struct DBError { pub message: String, diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 70e02164e08..484a1139bf9 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -17,6 +17,7 @@ use crate::metadata::{ PRUNING_CHECKPOINT_KEY, SCHEMA_VERSION_KEY, SPLIT_KEY, STATE_UPPER_LIMIT_NO_RETAIN, }; use crate::metrics; +use crate::state_cache::{PutStateOutcome, StateCache}; use crate::{ get_key_for_col, ChunkWriter, DBColumn, DatabaseBlock, Error, ItemStore, KeyValueStoreOp, PartialBeaconState, StoreItem, StoreOp, @@ -30,7 +31,8 @@ use slog::{debug, error, info, trace, warn, Logger}; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use state_processing::{ - BlockProcessingError, BlockReplayer, SlotProcessingError, StateProcessingStrategy, + block_replayer::PreSlotHook, AllCaches, BlockProcessingError, BlockReplayer, + SlotProcessingError, }; use std::cmp::min; use std::marker::PhantomData; @@ -66,12 +68,16 @@ pub struct HotColdDB, Cold: ItemStore> { pub hot_db: Hot, /// LRU cache of deserialized blocks and blobs. Updated whenever a block or blob is loaded. block_cache: Mutex>, + /// Cache of beacon states. + /// + /// LOCK ORDERING: this lock must always be locked *after* the `split` if both are required. + state_cache: Mutex>, /// LRU cache of replayed states. - state_cache: Mutex>>, + historic_state_cache: Mutex>>, /// Chain spec. pub(crate) spec: ChainSpec, /// Logger. - pub(crate) log: Logger, + pub log: Logger, /// Mere vessel for E. _phantom: PhantomData, } @@ -178,7 +184,8 @@ impl HotColdDB, MemoryStore> { blobs_db: MemoryStore::open(), hot_db: MemoryStore::open(), block_cache: Mutex::new(BlockCache::new(config.block_cache_size)), - state_cache: Mutex::new(LruCache::new(config.historic_state_cache_size)), + state_cache: Mutex::new(StateCache::new(config.state_cache_size)), + historic_state_cache: Mutex::new(LruCache::new(config.historic_state_cache_size)), config, spec, log, @@ -192,8 +199,6 @@ impl HotColdDB, MemoryStore> { impl HotColdDB, LevelDB> { /// Open a new or existing database, with the given paths to the hot and cold DBs. /// - /// The `slots_per_restore_point` parameter must be a divisor of `SLOTS_PER_HISTORICAL_ROOT`. - /// /// The `migrate_schema` function is passed in so that the parent `BeaconChain` can provide /// context and access `BeaconChain`-level code without creating a circular dependency. pub fn open( @@ -215,7 +220,8 @@ impl HotColdDB, LevelDB> { blobs_db: LevelDB::open(blobs_db_path)?, hot_db: LevelDB::open(hot_path)?, block_cache: Mutex::new(BlockCache::new(config.block_cache_size)), - state_cache: Mutex::new(LruCache::new(config.historic_state_cache_size)), + state_cache: Mutex::new(StateCache::new(config.state_cache_size)), + historic_state_cache: Mutex::new(LruCache::new(config.historic_state_cache_size)), config, spec, log, @@ -352,6 +358,21 @@ impl HotColdDB, LevelDB> { } impl, Cold: ItemStore> HotColdDB { + pub fn update_finalized_state( + &self, + state_root: Hash256, + block_root: Hash256, + state: BeaconState, + ) -> Result<(), Error> { + self.state_cache + .lock() + .update_finalized_state(state_root, block_root, state) + } + + pub fn state_cache_len(&self) -> usize { + self.state_cache.lock().len() + } + /// Store a block and update the LRU cache. pub fn put_block( &self, @@ -615,11 +636,26 @@ impl, Cold: ItemStore> HotColdDB /// Store a state in the store. pub fn put_state(&self, state_root: &Hash256, state: &BeaconState) -> Result<(), Error> { + self.put_state_possibly_temporary(state_root, state, false) + } + + /// Store a state in the store. + /// + /// The `temporary` flag indicates whether this state should be considered canonical. + pub fn put_state_possibly_temporary( + &self, + state_root: &Hash256, + state: &BeaconState, + temporary: bool, + ) -> Result<(), Error> { let mut ops: Vec = Vec::new(); if state.slot() < self.get_split_slot() { self.store_cold_state(state_root, state, &mut ops)?; self.cold_db.do_atomically(ops) } else { + if temporary { + ops.push(TemporaryFlag.as_kv_store_op(*state_root)); + } self.store_hot_state(state_root, state, &mut ops)?; self.hot_db.do_atomically(ops) } @@ -648,45 +684,16 @@ impl, Cold: ItemStore> HotColdDB // chain. This way we avoid returning a state that doesn't match `state_root`. self.load_cold_state(state_root) } else { - self.load_hot_state(state_root, StateProcessingStrategy::Accurate) + self.get_hot_state(state_root) } } else { - match self.load_hot_state(state_root, StateProcessingStrategy::Accurate)? { + match self.get_hot_state(state_root)? { Some(state) => Ok(Some(state)), None => self.load_cold_state(state_root), } } } - /// Fetch a state from the store, but don't compute all of the values when replaying blocks - /// upon that state (e.g., state roots). Additionally, only states from the hot store are - /// returned. - /// - /// See `Self::get_advanced_hot_state` for information about `max_slot`. - /// - /// ## Warning - /// - /// The returned state **is not a valid beacon state**, it can only be used for obtaining - /// shuffling to process attestations. At least the following components of the state will be - /// broken/invalid: - /// - /// - `state.state_roots` - /// - `state.block_roots` - pub fn get_inconsistent_state_for_attestation_verification_only( - &self, - block_root: &Hash256, - max_slot: Slot, - state_root: Hash256, - ) -> Result)>, Error> { - metrics::inc_counter(&metrics::BEACON_STATE_GET_COUNT); - self.get_advanced_hot_state_with_strategy( - *block_root, - max_slot, - state_root, - StateProcessingStrategy::Inconsistent, - ) - } - /// Get a state with `latest_block_root == block_root` advanced through to at most `max_slot`. /// /// The `state_root` argument is used to look up the block's un-advanced state in case an @@ -697,35 +704,29 @@ impl, Cold: ItemStore> HotColdDB /// - `result_state_root == state.canonical_root()` /// - `state.slot() <= max_slot` /// - `state.get_latest_block_root(result_state_root) == block_root` - /// - /// Presently this is only used to avoid loading the un-advanced split state, but in future will - /// be expanded to return states from an in-memory cache. pub fn get_advanced_hot_state( &self, block_root: Hash256, max_slot: Slot, state_root: Hash256, ) -> Result)>, Error> { - self.get_advanced_hot_state_with_strategy( - block_root, - max_slot, - state_root, - StateProcessingStrategy::Accurate, - ) - } + if let Some(cached) = self.get_advanced_hot_state_from_cache(block_root, max_slot) { + return Ok(Some(cached)); + } - /// Same as `get_advanced_hot_state` but taking a `StateProcessingStrategy`. - pub fn get_advanced_hot_state_with_strategy( - &self, - block_root: Hash256, - max_slot: Slot, - state_root: Hash256, - state_processing_strategy: StateProcessingStrategy, - ) -> Result)>, Error> { // Hold a read lock on the split point so it can't move while we're trying to load the // state. let split = self.split.read_recursive(); + if state_root != split.state_root { + warn!( + self.log, + "State cache missed"; + "state_root" => ?state_root, + "block_root" => ?block_root, + ); + } + // Sanity check max-slot against the split slot. if max_slot < split.slot { return Err(HotColdDBError::FinalizedStateNotInHotDatabase { @@ -741,11 +742,40 @@ impl, Cold: ItemStore> HotColdDB } else { state_root }; - let state = self - .load_hot_state(&state_root, state_processing_strategy)? - .map(|state| (state_root, state)); + let mut opt_state = self + .load_hot_state(&state_root)? + .map(|(state, _block_root)| (state_root, state)); + + if let Some((state_root, state)) = opt_state.as_mut() { + state.update_tree_hash_cache()?; + state.build_all_caches(&self.spec)?; + self.state_cache + .lock() + .put_state(*state_root, block_root, state)?; + debug!( + self.log, + "Cached state"; + "state_root" => ?state_root, + "slot" => state.slot(), + ); + } drop(split); - Ok(state) + Ok(opt_state) + } + + /// Same as `get_advanced_hot_state` but will return `None` if no compatible state is cached. + /// + /// If this function returns `Some(state)` then that `state` will always have + /// `latest_block_header` matching `block_root` but may not be advanced all the way through to + /// `max_slot`. + pub fn get_advanced_hot_state_from_cache( + &self, + block_root: Hash256, + max_slot: Slot, + ) -> Option<(Hash256, BeaconState)> { + self.state_cache + .lock() + .get_by_block_root(block_root, max_slot) } /// Delete a state, ensuring it is removed from the LRU cache, as well as from on-disk. @@ -755,17 +785,10 @@ impl, Cold: ItemStore> HotColdDB /// (which are frozen, and won't be deleted), or valid descendents of the finalized checkpoint /// (which will be deleted by this function but shouldn't be). pub fn delete_state(&self, state_root: &Hash256, slot: Slot) -> Result<(), Error> { - // Delete the state summary. - self.hot_db - .key_delete(DBColumn::BeaconStateSummary.into(), state_root.as_bytes())?; - - // Delete the full state if it lies on an epoch boundary. - if slot % E::slots_per_epoch() == 0 { - self.hot_db - .key_delete(DBColumn::BeaconState.into(), state_root.as_bytes())?; - } - - Ok(()) + self.do_atomically_with_block_and_blobs_cache(vec![StoreOp::DeleteState( + *state_root, + Some(slot), + )]) } pub fn forwards_block_roots_iterator( @@ -833,17 +856,9 @@ impl, Cold: ItemStore> HotColdDB }) = self.load_hot_state_summary(state_root)? { // NOTE: minor inefficiency here because we load an unnecessary hot state summary - // - // `StateProcessingStrategy` should be irrelevant here since we never replay blocks for an epoch - // boundary state in the hot DB. - let state = self - .load_hot_state( - &epoch_boundary_state_root, - StateProcessingStrategy::Accurate, - )? - .ok_or(HotColdDBError::MissingEpochBoundaryState( - epoch_boundary_state_root, - ))?; + let (state, _) = self.load_hot_state(&epoch_boundary_state_root)?.ok_or( + HotColdDBError::MissingEpochBoundaryState(epoch_boundary_state_root), + )?; Ok(Some(state)) } else { // Try the cold DB @@ -1029,11 +1044,14 @@ impl, Cold: ItemStore> HotColdDB StoreOp::DeleteBlock(block_root) => { guard.delete_block(&block_root); + self.state_cache.lock().delete_block_states(&block_root); } - StoreOp::DeleteBlobs(_) => (), + StoreOp::DeleteState(state_root, _) => { + self.state_cache.lock().delete_state(&state_root) + } - StoreOp::DeleteState(_, _) => (), + StoreOp::DeleteBlobs(_) => (), StoreOp::DeleteExecutionPayload(_) => (), @@ -1070,6 +1088,26 @@ impl, Cold: ItemStore> HotColdDB state: &BeaconState, ops: &mut Vec, ) -> Result<(), Error> { + // Put the state in the cache. + let block_root = state.get_latest_block_root(*state_root); + + // Avoid storing states in the database if they already exist in the state cache. + // The exception to this is the finalized state, which must exist in the cache before it + // is stored on disk. + if let PutStateOutcome::Duplicate = + self.state_cache + .lock() + .put_state(*state_root, block_root, state)? + { + debug!( + self.log, + "Skipping storage of cached state"; + "slot" => state.slot(), + "state_root" => ?state_root + ); + return Ok(()); + } + // On the epoch boundary, store the full state. if state.slot() % E::slots_per_epoch() == 0 { trace!( @@ -1091,14 +1129,51 @@ impl, Cold: ItemStore> HotColdDB Ok(()) } + /// Get a post-finalization state from the database or store. + pub fn get_hot_state(&self, state_root: &Hash256) -> Result>, Error> { + if let Some(state) = self.state_cache.lock().get_by_state_root(*state_root) { + return Ok(Some(state)); + } + + if *state_root != self.get_split_info().state_root { + // Do not warn on start up when loading the split state. + warn!( + self.log, + "State cache missed"; + "state_root" => ?state_root, + ); + } + + let state_from_disk = self.load_hot_state(state_root)?; + + if let Some((mut state, block_root)) = state_from_disk { + state.update_tree_hash_cache()?; + state.build_all_caches(&self.spec)?; + self.state_cache + .lock() + .put_state(*state_root, block_root, &state)?; + debug!( + self.log, + "Cached state"; + "state_root" => ?state_root, + "slot" => state.slot(), + ); + Ok(Some(state)) + } else { + Ok(None) + } + } + /// Load a post-finalization state from the hot database. /// /// Will replay blocks from the nearest epoch boundary. + /// + /// Return the `(state, latest_block_root)` where `latest_block_root` is the root of the last + /// block applied to `state`. pub fn load_hot_state( &self, state_root: &Hash256, - state_processing_strategy: StateProcessingStrategy, - ) -> Result>, Error> { + ) -> Result, Hash256)>, Error> { metrics::inc_counter(&metrics::BEACON_STATE_HOT_GET_COUNT); // If the state is marked as temporary, do not return it. It will become visible @@ -1113,16 +1188,47 @@ impl, Cold: ItemStore> HotColdDB epoch_boundary_state_root, }) = self.load_hot_state_summary(state_root)? { - let boundary_state = + let mut boundary_state = get_full_state(&self.hot_db, &epoch_boundary_state_root, &self.spec)?.ok_or( HotColdDBError::MissingEpochBoundaryState(epoch_boundary_state_root), )?; + // Immediately rebase the state from disk on the finalized state so that we can reuse + // parts of the tree for state root calculation in `replay_blocks`. + self.state_cache + .lock() + .rebase_on_finalized(&mut boundary_state, &self.spec)?; + // Optimization to avoid even *thinking* about replaying blocks if we're already // on an epoch boundary. - let state = if slot % E::slots_per_epoch() == 0 { + let mut state = if slot % E::slots_per_epoch() == 0 { boundary_state } else { + // Cache ALL intermediate states that are reached during block replay. We may want + // to restrict this in future to only cache epoch boundary states. At worst we will + // cache up to 32 states for each state loaded, which should not flush out the cache + // entirely. + let state_cache_hook = |state_root, state: &mut BeaconState| { + // Ensure all caches are built before attempting to cache. + state.update_tree_hash_cache()?; + state.build_all_caches(&self.spec)?; + + let latest_block_root = state.get_latest_block_root(state_root); + let state_slot = state.slot(); + if let PutStateOutcome::New = + self.state_cache + .lock() + .put_state(state_root, latest_block_root, state)? + { + debug!( + self.log, + "Cached ancestor state"; + "state_root" => ?state_root, + "slot" => state_slot, + ); + } + Ok(()) + }; let blocks = self.load_blocks_to_replay(boundary_state.slot(), slot, latest_block_root)?; self.replay_blocks( @@ -1130,11 +1236,12 @@ impl, Cold: ItemStore> HotColdDB blocks, slot, no_state_root_iter(), - state_processing_strategy, + Some(Box::new(state_cache_hook)), )? }; + state.apply_pending_mutations()?; - Ok(Some(state)) + Ok(Some((state, latest_block_root))) } else { Ok(None) } @@ -1233,7 +1340,9 @@ impl, Cold: ItemStore> HotColdDB partial_state.load_randao_mixes(&self.cold_db, &self.spec)?; partial_state.load_historical_summaries(&self.cold_db, &self.spec)?; - partial_state.try_into() + let mut state: BeaconState = partial_state.try_into()?; + state.apply_pending_mutations()?; + Ok(state) } /// Load a restore point state by its `restore_point_index`. @@ -1247,7 +1356,7 @@ impl, Cold: ItemStore> HotColdDB /// Load a frozen state that lies between restore points. fn load_cold_intermediate_state(&self, slot: Slot) -> Result, Error> { - if let Some(state) = self.state_cache.lock().get(&slot) { + if let Some(state) = self.historic_state_cache.lock().get(&slot) { return Ok(state.clone()); } @@ -1261,7 +1370,7 @@ impl, Cold: ItemStore> HotColdDB let mut low_state: Option> = None; // Try to get a more recent state from the cache to avoid massive blocks replay. - for (s, state) in self.state_cache.lock().iter() { + for (s, state) in self.historic_state_cache.lock().iter() { if s.as_u64() / self.config.slots_per_restore_point == low_restore_point_idx && *s < slot && low_slot < *s @@ -1299,16 +1408,11 @@ impl, Cold: ItemStore> HotColdDB &self.spec, )?; - let state = self.replay_blocks( - low_state, - blocks, - slot, - Some(state_root_iter), - StateProcessingStrategy::Accurate, - )?; + let mut state = self.replay_blocks(low_state, blocks, slot, Some(state_root_iter), None)?; + state.apply_pending_mutations()?; // If state is not error, put it in the cache. - self.state_cache.lock().put(slot, state.clone()); + self.historic_state_cache.lock().put(slot, state.clone()); Ok(state) } @@ -1390,16 +1494,15 @@ impl, Cold: ItemStore> HotColdDB /// /// Will skip slots as necessary. The returned state is not guaranteed /// to have any caches built, beyond those immediately required by block processing. - fn replay_blocks( + pub fn replay_blocks( &self, state: BeaconState, blocks: Vec>>, target_slot: Slot, state_root_iter: Option>>, - state_processing_strategy: StateProcessingStrategy, + pre_slot_hook: Option>, ) -> Result, Error> { let mut block_replayer = BlockReplayer::new(state, &self.spec) - .state_processing_strategy(state_processing_strategy) .no_signature_verification() .minimal_block_root_verification(); @@ -1408,17 +1511,20 @@ impl, Cold: ItemStore> HotColdDB block_replayer = block_replayer.state_root_iter(state_root_iter); } + if let Some(pre_slot_hook) = pre_slot_hook { + block_replayer = block_replayer.pre_slot_hook(pre_slot_hook); + } + block_replayer .apply_blocks(blocks, Some(target_slot)) .map(|block_replayer| { if have_state_root_iterator && block_replayer.state_root_miss() { warn!( self.log, - "State root iterator miss"; + "State root cache miss during block replay"; "slot" => target_slot, ); } - block_replayer.into_state() }) } @@ -2213,7 +2319,7 @@ impl, Cold: ItemStore> HotColdDB } /// This function fills in missing block roots between last restore point slot and split - /// slot, if any. + /// slot, if any. pub fn heal_freezer_block_roots_at_split(&self) -> Result<(), Error> { let split = self.get_split_info(); let last_restore_point_slot = (split.slot - 1) / self.config.slots_per_restore_point @@ -2528,15 +2634,22 @@ pub fn migrate_database, Cold: ItemStore>( }; store.hot_db.put_sync(&SPLIT_KEY, &split)?; - // Split point is now persisted in the hot database on disk. The in-memory split point - // hasn't been modified elsewhere since we keep a write lock on it. It's safe to update + // Split point is now persisted in the hot database on disk. The in-memory split point + // hasn't been modified elsewhere since we keep a write lock on it. It's safe to update // the in-memory split point now. *split_guard = split; } - // Delete the states from the hot database if we got this far. + // Delete the blocks and states from the hot database if we got this far. store.do_atomically_with_block_and_blobs_cache(hot_db_ops)?; + // Update the cache's view of the finalized state. + store.update_finalized_state( + finalized_state_root, + finalized_block_root, + finalized_state.clone(), + )?; + debug!( store.log, "Freezer migration complete"; diff --git a/beacon_node/store/src/impls/beacon_state.rs b/beacon_node/store/src/impls/beacon_state.rs index d08bf564927..f752bf39795 100644 --- a/beacon_node/store/src/impls/beacon_state.rs +++ b/beacon_node/store/src/impls/beacon_state.rs @@ -46,14 +46,14 @@ pub fn get_full_state, E: EthSpec>( #[derive(Encode)] pub struct StorageContainer { state: BeaconState, - committee_caches: Vec, + committee_caches: Vec>, } impl StorageContainer { /// Create a new instance for storing a `BeaconState`. pub fn new(state: &BeaconState) -> Self { Self { - state: state.clone_with(CloneConfig::none()), + state: state.clone(), committee_caches: state.committee_caches().to_vec(), } } diff --git a/beacon_node/store/src/iter.rs b/beacon_node/store/src/iter.rs index e459c1c3575..03090ca14c5 100644 --- a/beacon_node/store/src/iter.rs +++ b/beacon_node/store/src/iter.rs @@ -412,15 +412,16 @@ mod test { let mut hashes = (0..).map(Hash256::from_low_u64_be); let roots_a = state_a.block_roots_mut(); for i in 0..roots_a.len() { - roots_a[i] = hashes.next().unwrap() + *roots_a.get_mut(i).unwrap() = hashes.next().unwrap(); } let roots_b = state_b.block_roots_mut(); for i in 0..roots_b.len() { - roots_b[i] = hashes.next().unwrap() + *roots_b.get_mut(i).unwrap() = hashes.next().unwrap(); } let state_a_root = hashes.next().unwrap(); - state_b.state_roots_mut()[0] = state_a_root; + *state_b.state_roots_mut().get_mut(0).unwrap() = state_a_root; + state_a.apply_pending_mutations().unwrap(); store.put_state(&state_a_root, &state_a).unwrap(); let iter = BlockRootsIterator::new(&store, &state_b); @@ -472,6 +473,9 @@ mod test { let state_a_root = Hash256::from_low_u64_be(slots_per_historical_root as u64); let state_b_root = Hash256::from_low_u64_be(slots_per_historical_root as u64 * 2); + state_a.apply_pending_mutations().unwrap(); + state_b.apply_pending_mutations().unwrap(); + store.put_state(&state_a_root, &state_a).unwrap(); store.put_state(&state_b_root, &state_b).unwrap(); diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index c3136a910db..66032d89c52 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -26,6 +26,7 @@ pub mod metadata; pub mod metrics; mod partial_beacon_state; pub mod reconstruct; +pub mod state_cache; pub mod iter; diff --git a/beacon_node/store/src/partial_beacon_state.rs b/beacon_node/store/src/partial_beacon_state.rs index 4e5a2b8e64b..25438fc7e0a 100644 --- a/beacon_node/store/src/partial_beacon_state.rs +++ b/beacon_node/store/src/partial_beacon_state.rs @@ -34,42 +34,42 @@ where pub latest_block_header: BeaconBlockHeader, #[ssz(skip_serializing, skip_deserializing)] - pub block_roots: Option>, + pub block_roots: Option>, #[ssz(skip_serializing, skip_deserializing)] - pub state_roots: Option>, + pub state_roots: Option>, #[ssz(skip_serializing, skip_deserializing)] - pub historical_roots: Option>, + pub historical_roots: Option>, // Ethereum 1.0 chain data pub eth1_data: Eth1Data, - pub eth1_data_votes: VariableList, + pub eth1_data_votes: List, pub eth1_deposit_index: u64, // Registry - pub validators: VariableList, - pub balances: VariableList, + pub validators: List, + pub balances: List, // Shuffling /// Randao value from the current slot, for patching into the per-epoch randao vector. pub latest_randao_value: Hash256, #[ssz(skip_serializing, skip_deserializing)] - pub randao_mixes: Option>, + pub randao_mixes: Option>, // Slashings - slashings: FixedVector, + slashings: Vector, // Attestations (genesis fork only) #[superstruct(only(Base))] - pub previous_epoch_attestations: VariableList, E::MaxPendingAttestations>, + pub previous_epoch_attestations: List, E::MaxPendingAttestations>, #[superstruct(only(Base))] - pub current_epoch_attestations: VariableList, E::MaxPendingAttestations>, + pub current_epoch_attestations: List, E::MaxPendingAttestations>, // Participation (Altair and later) #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] - pub previous_epoch_participation: VariableList, + pub previous_epoch_participation: List, #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] - pub current_epoch_participation: VariableList, + pub current_epoch_participation: List, // Finality pub justification_bits: BitVector, @@ -79,7 +79,7 @@ where // Inactivity #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] - pub inactivity_scores: VariableList, + pub inactivity_scores: List, // Light-client sync committees #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] @@ -117,7 +117,7 @@ where #[ssz(skip_serializing, skip_deserializing)] #[superstruct(only(Capella, Deneb, Electra))] - pub historical_summaries: Option>, + pub historical_summaries: Option>, } /// Implement the conversion function from BeaconState -> PartialBeaconState. @@ -369,7 +369,9 @@ impl PartialBeaconState { // Patch the value for the current slot into the index for the current epoch let current_epoch = self.slot().epoch(E::slots_per_epoch()); let len = randao_mixes.len(); - randao_mixes[current_epoch.as_usize() % len] = *self.latest_randao_value(); + *randao_mixes + .get_mut(current_epoch.as_usize() % len) + .ok_or(Error::RandaoMixOutOfBounds)? = *self.latest_randao_value(); *self.randao_mixes_mut() = Some(randao_mixes) } @@ -422,7 +424,6 @@ macro_rules! impl_try_into_beacon_state { exit_cache: <_>::default(), slashings_cache: <_>::default(), epoch_cache: <_>::default(), - tree_hash_cache: <_>::default(), // Variant-specific fields $( diff --git a/beacon_node/store/src/reconstruct.rs b/beacon_node/store/src/reconstruct.rs index 8fe13777ac4..8ef4886565c 100644 --- a/beacon_node/store/src/reconstruct.rs +++ b/beacon_node/store/src/reconstruct.rs @@ -5,7 +5,7 @@ use itertools::{process_results, Itertools}; use slog::info; use state_processing::{ per_block_processing, per_slot_processing, BlockSignatureStrategy, ConsensusContext, - StateProcessingStrategy, VerifyBlockRoot, + VerifyBlockRoot, }; use std::sync::Arc; use types::{EthSpec, Hash256}; @@ -94,7 +94,6 @@ where &mut state, &block, BlockSignatureStrategy::NoVerification, - StateProcessingStrategy::Accurate, VerifyBlockRoot::True, &mut ctxt, &self.spec, diff --git a/beacon_node/store/src/state_cache.rs b/beacon_node/store/src/state_cache.rs new file mode 100644 index 00000000000..5c1faa7f2fd --- /dev/null +++ b/beacon_node/store/src/state_cache.rs @@ -0,0 +1,303 @@ +use crate::Error; +use lru::LruCache; +use std::collections::{BTreeMap, HashMap, HashSet}; +use std::num::NonZeroUsize; +use types::{BeaconState, ChainSpec, Epoch, EthSpec, Hash256, Slot}; + +/// Fraction of the LRU cache to leave intact during culling. +const CULL_EXEMPT_NUMERATOR: usize = 1; +const CULL_EXEMPT_DENOMINATOR: usize = 10; + +/// States that are less than or equal to this many epochs old *could* become finalized and will not +/// be culled from the cache. +const EPOCH_FINALIZATION_LIMIT: u64 = 4; + +#[derive(Debug)] +pub struct FinalizedState { + state_root: Hash256, + state: BeaconState, +} + +/// Map from block_root -> slot -> state_root. +#[derive(Debug, Default)] +pub struct BlockMap { + blocks: HashMap, +} + +/// Map from slot -> state_root. +#[derive(Debug, Default)] +pub struct SlotMap { + slots: BTreeMap, +} + +#[derive(Debug)] +pub struct StateCache { + finalized_state: Option>, + states: LruCache>, + block_map: BlockMap, + max_epoch: Epoch, +} + +#[derive(Debug)] +pub enum PutStateOutcome { + Finalized, + Duplicate, + New, +} + +#[allow(clippy::len_without_is_empty)] +impl StateCache { + pub fn new(capacity: NonZeroUsize) -> Self { + StateCache { + finalized_state: None, + states: LruCache::new(capacity), + block_map: BlockMap::default(), + max_epoch: Epoch::new(0), + } + } + + pub fn len(&self) -> usize { + self.states.len() + } + + pub fn capacity(&self) -> usize { + self.states.cap().get() + } + + pub fn update_finalized_state( + &mut self, + state_root: Hash256, + block_root: Hash256, + state: BeaconState, + ) -> Result<(), Error> { + if state.slot() % E::slots_per_epoch() != 0 { + return Err(Error::FinalizedStateUnaligned); + } + + if self + .finalized_state + .as_ref() + .map_or(false, |finalized_state| { + state.slot() < finalized_state.state.slot() + }) + { + return Err(Error::FinalizedStateDecreasingSlot); + } + + // Add to block map. + self.block_map.insert(block_root, state.slot(), state_root); + + // Prune block map. + let state_roots_to_prune = self.block_map.prune(state.slot()); + + // Delete states. + for state_root in state_roots_to_prune { + self.states.pop(&state_root); + } + + // Update finalized state. + self.finalized_state = Some(FinalizedState { state_root, state }); + Ok(()) + } + + /// Rebase the given state on the finalized state in order to reduce its memory consumption. + /// + /// This function should only be called on states that are likely not to already share tree + /// nodes with the finalized state, e.g. states loaded from disk. + /// + /// If the finalized state is not initialized this function is a no-op. + pub fn rebase_on_finalized( + &self, + state: &mut BeaconState, + spec: &ChainSpec, + ) -> Result<(), Error> { + if let Some(finalized_state) = &self.finalized_state { + state.rebase_on(&finalized_state.state, spec)?; + } + Ok(()) + } + + /// Return a status indicating whether the state already existed in the cache. + pub fn put_state( + &mut self, + state_root: Hash256, + block_root: Hash256, + state: &BeaconState, + ) -> Result { + if self + .finalized_state + .as_ref() + .map_or(false, |finalized_state| { + finalized_state.state_root == state_root + }) + { + return Ok(PutStateOutcome::Finalized); + } + + if self.states.peek(&state_root).is_some() { + return Ok(PutStateOutcome::Duplicate); + } + + // Refuse states with pending mutations: we want cached states to be as small as possible + // i.e. stored entirely as a binary merkle tree with no updates overlaid. + if state.has_pending_mutations() { + return Err(Error::StateForCacheHasPendingUpdates { + state_root, + slot: state.slot(), + }); + } + + // Update the cache's idea of the max epoch. + self.max_epoch = std::cmp::max(state.current_epoch(), self.max_epoch); + + // If the cache is full, use the custom cull routine to make room. + if let Some(over_capacity) = self.len().checked_sub(self.capacity()) { + self.cull(over_capacity + 1); + } + + // Insert the full state into the cache. + self.states.put(state_root, state.clone()); + + // Record the connection from block root and slot to this state. + let slot = state.slot(); + self.block_map.insert(block_root, slot, state_root); + + Ok(PutStateOutcome::New) + } + + pub fn get_by_state_root(&mut self, state_root: Hash256) -> Option> { + if let Some(ref finalized_state) = self.finalized_state { + if state_root == finalized_state.state_root { + return Some(finalized_state.state.clone()); + } + } + self.states.get(&state_root).cloned() + } + + pub fn get_by_block_root( + &mut self, + block_root: Hash256, + slot: Slot, + ) -> Option<(Hash256, BeaconState)> { + let slot_map = self.block_map.blocks.get(&block_root)?; + + // Find the state at `slot`, or failing that the most recent ancestor. + let state_root = slot_map + .slots + .iter() + .rev() + .find_map(|(ancestor_slot, state_root)| { + (*ancestor_slot <= slot).then_some(*state_root) + })?; + + let state = self.get_by_state_root(state_root)?; + Some((state_root, state)) + } + + pub fn delete_state(&mut self, state_root: &Hash256) { + self.states.pop(state_root); + self.block_map.delete(state_root); + } + + pub fn delete_block_states(&mut self, block_root: &Hash256) { + if let Some(slot_map) = self.block_map.delete_block_states(block_root) { + for state_root in slot_map.slots.values() { + self.states.pop(state_root); + } + } + } + + /// Cull approximately `count` states from the cache. + /// + /// States are culled LRU, with the following extra order imposed: + /// + /// - Advanced states. + /// - Mid-epoch unadvanced states. + /// - Epoch-boundary states that are too old to be finalized. + /// - Epoch-boundary states that could be finalized. + pub fn cull(&mut self, count: usize) { + let cull_exempt = std::cmp::max( + 1, + self.len() * CULL_EXEMPT_NUMERATOR / CULL_EXEMPT_DENOMINATOR, + ); + + // Stage 1: gather states to cull. + let mut advanced_state_roots = vec![]; + let mut mid_epoch_state_roots = vec![]; + let mut old_boundary_state_roots = vec![]; + let mut good_boundary_state_roots = vec![]; + for (&state_root, state) in self.states.iter().skip(cull_exempt) { + let is_advanced = state.slot() > state.latest_block_header().slot; + let is_boundary = state.slot() % E::slots_per_epoch() == 0; + let could_finalize = + (self.max_epoch - state.current_epoch()) <= EPOCH_FINALIZATION_LIMIT; + + if is_boundary { + if could_finalize { + good_boundary_state_roots.push(state_root); + } else { + old_boundary_state_roots.push(state_root); + } + } else if is_advanced { + advanced_state_roots.push(state_root); + } else { + mid_epoch_state_roots.push(state_root); + } + + // Terminate early in the common case where we've already found enough junk to cull. + if advanced_state_roots.len() == count { + break; + } + } + + // Stage 2: delete. + // This could probably be more efficient in how it interacts with the block map. + for state_root in advanced_state_roots + .iter() + .chain(mid_epoch_state_roots.iter()) + .chain(old_boundary_state_roots.iter()) + .chain(good_boundary_state_roots.iter()) + .take(count) + { + self.delete_state(state_root); + } + } +} + +impl BlockMap { + fn insert(&mut self, block_root: Hash256, slot: Slot, state_root: Hash256) { + let slot_map = self.blocks.entry(block_root).or_default(); + slot_map.slots.insert(slot, state_root); + } + + fn prune(&mut self, finalized_slot: Slot) -> HashSet { + let mut pruned_states = HashSet::new(); + + self.blocks.retain(|_, slot_map| { + slot_map.slots.retain(|slot, state_root| { + let keep = *slot >= finalized_slot; + if !keep { + pruned_states.insert(*state_root); + } + keep + }); + + !slot_map.slots.is_empty() + }); + + pruned_states + } + + fn delete(&mut self, state_root_to_delete: &Hash256) { + self.blocks.retain(|_, slot_map| { + slot_map + .slots + .retain(|_, state_root| state_root != state_root_to_delete); + !slot_map.slots.is_empty() + }); + } + + fn delete_block_states(&mut self, block_root: &Hash256) -> Option { + self.blocks.remove(block_root) + } +} diff --git a/common/task_executor/Cargo.toml b/common/task_executor/Cargo.toml index b3d58fa5ea8..cc9a2c5097b 100644 --- a/common/task_executor/Cargo.toml +++ b/common/task_executor/Cargo.toml @@ -12,3 +12,4 @@ futures = { workspace = true } lazy_static = { workspace = true } lighthouse_metrics = { workspace = true } sloggers = { workspace = true } +logging = { workspace = true } diff --git a/common/task_executor/src/test_utils.rs b/common/task_executor/src/test_utils.rs index 6e372d97575..ec8f45d850e 100644 --- a/common/task_executor/src/test_utils.rs +++ b/common/task_executor/src/test_utils.rs @@ -1,4 +1,5 @@ use crate::TaskExecutor; +use logging::test_logger; use slog::Logger; use sloggers::{null::NullLoggerBuilder, Build}; use std::sync::Arc; @@ -26,7 +27,7 @@ impl Default for TestRuntime { fn default() -> Self { let (runtime_shutdown, exit) = async_channel::bounded(1); let (shutdown_tx, _) = futures::channel::mpsc::channel(1); - let log = null_logger().unwrap(); + let log = test_logger(); let (runtime, handle) = if let Ok(handle) = runtime::Handle::try_current() { (None, handle) diff --git a/consensus/state_processing/src/all_caches.rs b/consensus/state_processing/src/all_caches.rs index 106692c63aa..b915091405b 100644 --- a/consensus/state_processing/src/all_caches.rs +++ b/consensus/state_processing/src/all_caches.rs @@ -9,12 +9,14 @@ use types::{BeaconState, ChainSpec, EpochCacheError, EthSpec, Hash256, RelativeE pub trait AllCaches { /// Build all caches. /// - /// Note that this excludes the tree-hash cache. That needs to be managed separately. + /// Note that this excludes milhouse's intrinsic tree-hash cache. That needs to be managed + /// separately. fn build_all_caches(&mut self, spec: &ChainSpec) -> Result<(), EpochCacheError>; /// Return true if all caches are built. /// - /// Note that this excludes the tree-hash cache. That needs to be managed separately. + /// Note that this excludes milhouse's intrinsic tree-hash cache. That needs to be managed + /// separately. fn all_caches_built(&self) -> bool; } diff --git a/consensus/state_processing/src/block_replayer.rs b/consensus/state_processing/src/block_replayer.rs index f502d7f692c..d7621ebf18b 100644 --- a/consensus/state_processing/src/block_replayer.rs +++ b/consensus/state_processing/src/block_replayer.rs @@ -6,19 +6,23 @@ use crate::{ use itertools::Itertools; use std::iter::Peekable; use std::marker::PhantomData; -use types::{BeaconState, BlindedPayload, ChainSpec, EthSpec, Hash256, SignedBeaconBlock, Slot}; +use types::{ + BeaconState, BeaconStateError, BlindedPayload, ChainSpec, EthSpec, Hash256, SignedBeaconBlock, + Slot, +}; -type PreBlockHook<'a, E, Error> = Box< +pub type PreBlockHook<'a, E, Error> = Box< dyn FnMut(&mut BeaconState, &SignedBeaconBlock>) -> Result<(), Error> + 'a, >; -type PostBlockHook<'a, E, Error> = PreBlockHook<'a, E, Error>; -type PreSlotHook<'a, E, Error> = Box) -> Result<(), Error> + 'a>; -type PostSlotHook<'a, E, Error> = Box< +pub type PostBlockHook<'a, E, Error> = PreBlockHook<'a, E, Error>; +pub type PreSlotHook<'a, E, Error> = + Box) -> Result<(), Error> + 'a>; +pub type PostSlotHook<'a, E, Error> = Box< dyn FnMut(&mut BeaconState, Option>, bool) -> Result<(), Error> + 'a, >; -type StateRootIterDefault = std::iter::Empty>; +pub type StateRootIterDefault = std::iter::Empty>; /// Efficiently apply blocks to a state while configuring various parameters. /// @@ -31,7 +35,6 @@ pub struct BlockReplayer< > { state: BeaconState, spec: &'a ChainSpec, - state_processing_strategy: StateProcessingStrategy, block_sig_strategy: BlockSignatureStrategy, verify_block_root: Option, pre_block_hook: Option>, @@ -45,9 +48,9 @@ pub struct BlockReplayer< #[derive(Debug)] pub enum BlockReplayError { - NoBlocks, SlotProcessing(SlotProcessingError), BlockProcessing(BlockProcessingError), + BeaconState(BeaconStateError), } impl From for BlockReplayError { @@ -62,14 +65,10 @@ impl From for BlockReplayError { } } -/// Defines how state roots should be computed and whether to perform all state transitions during block replay. -#[derive(PartialEq, Clone, Copy)] -pub enum StateProcessingStrategy { - /// Perform all transitions faithfully to the specification. - Accurate, - /// Don't compute state roots and process withdrawals, eventually computing an invalid beacon - /// state that can only be used for obtaining shuffling. - Inconsistent, +impl From for BlockReplayError { + fn from(e: BeaconStateError) -> Self { + Self::BeaconState(e) + } } impl<'a, E, Error, StateRootIter> BlockReplayer<'a, E, Error, StateRootIter> @@ -89,7 +88,6 @@ where Self { state, spec, - state_processing_strategy: StateProcessingStrategy::Accurate, block_sig_strategy: BlockSignatureStrategy::VerifyBulk, verify_block_root: Some(VerifyBlockRoot::True), pre_block_hook: None, @@ -102,18 +100,6 @@ where } } - /// Set the replayer's state processing strategy different from the default. - pub fn state_processing_strategy( - mut self, - state_processing_strategy: StateProcessingStrategy, - ) -> Self { - if state_processing_strategy == StateProcessingStrategy::Inconsistent { - self.verify_block_root = None; - } - self.state_processing_strategy = state_processing_strategy; - self - } - /// Set the replayer's block signature verification strategy. pub fn block_signature_strategy(mut self, block_sig_strategy: BlockSignatureStrategy) -> Self { self.block_sig_strategy = block_sig_strategy; @@ -175,21 +161,24 @@ where self } - /// Compute the state root for `slot` as efficiently as possible. + /// Compute the state root for `self.state` as efficiently as possible. + /// + /// This function MUST only be called when `self.state` is a post-state, i.e. it MUST not be + /// called between advancing a state with `per_slot_processing` and applying the block for that + /// slot. /// /// The `blocks` should be the full list of blocks being applied and `i` should be the index of /// the next block that will be applied, or `blocks.len()` if all blocks have already been /// applied. + /// + /// If the state root is not available from the state root iterator or the blocks then it will + /// be computed from `self.state` and a state root iterator miss will be recorded. fn get_state_root( &mut self, - slot: Slot, blocks: &[SignedBeaconBlock>], i: usize, - ) -> Result, Error> { - // If we don't care about state roots then return immediately. - if self.state_processing_strategy == StateProcessingStrategy::Inconsistent { - return Ok(Some(Hash256::zero())); - } + ) -> Result { + let slot = self.state.slot(); // If a state root iterator is configured, use it to find the root. if let Some(ref mut state_root_iter) = self.state_root_iter { @@ -199,7 +188,7 @@ where .transpose()?; if let Some((root, _)) = opt_root { - return Ok(Some(root)); + return Ok(root); } } @@ -207,13 +196,17 @@ where if let Some(prev_i) = i.checked_sub(1) { if let Some(prev_block) = blocks.get(prev_i) { if prev_block.slot() == slot { - return Ok(Some(prev_block.state_root())); + return Ok(prev_block.state_root()); } } } self.state_root_miss = true; - Ok(None) + let state_root = self + .state + .update_tree_hash_cache() + .map_err(BlockReplayError::from)?; + Ok(state_root) } /// Apply `blocks` atop `self.state`, taking care of slot processing. @@ -232,12 +225,13 @@ where } while self.state.slot() < block.slot() { + let state_root = self.get_state_root(&blocks, i)?; + if let Some(ref mut pre_slot_hook) = self.pre_slot_hook { - pre_slot_hook(&mut self.state)?; + pre_slot_hook(state_root, &mut self.state)?; } - let state_root = self.get_state_root(self.state.slot(), &blocks, i)?; - let summary = per_slot_processing(&mut self.state, state_root, self.spec) + let summary = per_slot_processing(&mut self.state, Some(state_root), self.spec) .map_err(BlockReplayError::from)?; if let Some(ref mut post_slot_hook) = self.post_slot_hook { @@ -250,15 +244,11 @@ where pre_block_hook(&mut self.state, block)?; } - let verify_block_root = self.verify_block_root.unwrap_or_else(|| { - // If no explicit policy is set, verify only the first 1 or 2 block roots if using - // accurate state roots. Inaccurate state roots require block root verification to - // be off. - if i <= 1 && self.state_processing_strategy == StateProcessingStrategy::Accurate { - VerifyBlockRoot::True - } else { - VerifyBlockRoot::False - } + // If no explicit policy is set, verify only the first 1 or 2 block roots. + let verify_block_root = self.verify_block_root.unwrap_or(if i <= 1 { + VerifyBlockRoot::True + } else { + VerifyBlockRoot::False }); // Proposer index was already checked when this block was originally processed, we // can omit recomputing it during replay. @@ -268,7 +258,6 @@ where &mut self.state, block, self.block_sig_strategy, - self.state_processing_strategy, verify_block_root, &mut ctxt, self.spec, @@ -282,12 +271,13 @@ where if let Some(target_slot) = target_slot { while self.state.slot() < target_slot { + let state_root = self.get_state_root(&blocks, blocks.len())?; + if let Some(ref mut pre_slot_hook) = self.pre_slot_hook { - pre_slot_hook(&mut self.state)?; + pre_slot_hook(state_root, &mut self.state)?; } - let state_root = self.get_state_root(self.state.slot(), &blocks, blocks.len())?; - let summary = per_slot_processing(&mut self.state, state_root, self.spec) + let summary = per_slot_processing(&mut self.state, Some(state_root), self.spec) .map_err(BlockReplayError::from)?; if let Some(ref mut post_slot_hook) = self.post_slot_hook { diff --git a/consensus/state_processing/src/common/initiate_validator_exit.rs b/consensus/state_processing/src/common/initiate_validator_exit.rs index 84656d9c890..a40a9dfd398 100644 --- a/consensus/state_processing/src/common/initiate_validator_exit.rs +++ b/consensus/state_processing/src/common/initiate_validator_exit.rs @@ -30,13 +30,14 @@ pub fn initiate_validator_exit( exit_queue_epoch.safe_add_assign(1)?; } - let validator = state.get_validator_mut(index)?; + let validator = state.get_validator_cow(index)?; // Return if the validator already initiated exit if validator.exit_epoch != spec.far_future_epoch { return Ok(()); } + let validator = validator.into_mut()?; validator.exit_epoch = exit_queue_epoch; validator.withdrawable_epoch = exit_queue_epoch.safe_add(spec.min_validator_withdrawability_delay)?; diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index 036ab23498c..5ab9a99a3b1 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -28,7 +28,7 @@ pub fn initialize_beacon_state_from_eth1( let mut state = BeaconState::new(genesis_time, eth1_data, spec); // Seed RANDAO with Eth1 entropy - state.fill_randao_mixes_with(eth1_block_hash); + state.fill_randao_mixes_with(eth1_block_hash)?; let mut deposit_tree = DepositDataTree::create(&[], 0, DEPOSIT_TREE_DEPTH); @@ -152,7 +152,9 @@ pub fn process_activations( spec: &ChainSpec, ) -> Result<(), Error> { let (validators, balances, _) = state.validators_and_balances_and_progressive_balances_mut(); - for (index, validator) in validators.iter_mut().enumerate() { + let mut validators_iter = validators.iter_cow(); + while let Some((index, validator)) = validators_iter.next_cow() { + let validator = validator.into_mut()?; let balance = balances .get(index) .copied() diff --git a/consensus/state_processing/src/lib.rs b/consensus/state_processing/src/lib.rs index 7d84c426e8c..74f9d84bb11 100644 --- a/consensus/state_processing/src/lib.rs +++ b/consensus/state_processing/src/lib.rs @@ -30,7 +30,7 @@ pub mod upgrade; pub mod verify_operation; pub use all_caches::AllCaches; -pub use block_replayer::{BlockReplayError, BlockReplayer, StateProcessingStrategy}; +pub use block_replayer::{BlockReplayError, BlockReplayer}; pub use consensus_context::{ConsensusContext, ContextError}; pub use genesis::{ eth2_genesis_time, initialize_beacon_state_from_eth1, is_valid_genesis_state, diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index b370ec6216b..a8447b7714a 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -40,7 +40,6 @@ mod verify_exit; mod verify_proposer_slashing; use crate::common::decrease_balance; -use crate::StateProcessingStrategy; use crate::common::update_progressive_balances_cache::{ initialize_progressive_balances_cache, update_progressive_balances_metrics, @@ -102,7 +101,6 @@ pub fn per_block_processing>( state: &mut BeaconState, signed_block: &SignedBeaconBlock, block_signature_strategy: BlockSignatureStrategy, - state_processing_strategy: StateProcessingStrategy, verify_block_root: VerifyBlockRoot, ctxt: &mut ConsensusContext, spec: &ChainSpec, @@ -172,9 +170,7 @@ pub fn per_block_processing>( // previous block. if is_execution_enabled(state, block.body()) { let body = block.body(); - if state_processing_strategy == StateProcessingStrategy::Accurate { - process_withdrawals::(state, body.execution_payload()?, spec)?; - } + process_withdrawals::(state, body.execution_payload()?, spec)?; process_execution_payload::(state, body, spec)?; } diff --git a/consensus/state_processing/src/per_block_processing/errors.rs b/consensus/state_processing/src/per_block_processing/errors.rs index 28d36dbc518..336895514f9 100644 --- a/consensus/state_processing/src/per_block_processing/errors.rs +++ b/consensus/state_processing/src/per_block_processing/errors.rs @@ -82,6 +82,7 @@ pub enum BlockProcessingError { }, ExecutionInvalid, ConsensusContext(ContextError), + MilhouseError(milhouse::Error), EpochCacheError(EpochCacheError), WithdrawalsRootMismatch { expected: Hash256, @@ -138,6 +139,12 @@ impl From for BlockProcessingError { } } +impl From for BlockProcessingError { + fn from(e: milhouse::Error) -> Self { + Self::MilhouseError(e) + } +} + impl From> for BlockProcessingError { fn from(e: BlockOperationError) -> BlockProcessingError { match e { diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 63b7c9e01fb..d812306a0f7 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -128,7 +128,7 @@ pub mod altair_deneb { let previous_epoch = ctxt.previous_epoch; let current_epoch = ctxt.current_epoch; - let attesting_indices = &verify_attestation_for_block_inclusion( + let attesting_indices = verify_attestation_for_block_inclusion( state, attestation, ctxt, @@ -136,7 +136,8 @@ pub mod altair_deneb { spec, ) .map_err(|e| e.into_with_index(att_index))? - .attesting_indices; + .attesting_indices + .clone(); // Matching roots, participation flag indices let data = &attestation.data; @@ -146,7 +147,7 @@ pub mod altair_deneb { // Update epoch participation flags. let mut proposer_reward_numerator = 0; - for index in attesting_indices { + for index in &attesting_indices { let index = *index as usize; let validator_effective_balance = state.epoch_cache().get_effective_balance(index)?; diff --git a/consensus/state_processing/src/per_block_processing/tests.rs b/consensus/state_processing/src/per_block_processing/tests.rs index 2a2b67e30da..f0055fa80dd 100644 --- a/consensus/state_processing/src/per_block_processing/tests.rs +++ b/consensus/state_processing/src/per_block_processing/tests.rs @@ -5,7 +5,7 @@ use crate::per_block_processing::errors::{ DepositInvalid, HeaderInvalid, IndexedAttestationInvalid, IntoWithIndex, ProposerSlashingInvalid, }; -use crate::{per_block_processing, BlockReplayError, BlockReplayer, StateProcessingStrategy}; +use crate::{per_block_processing, BlockReplayError, BlockReplayer}; use crate::{ per_block_processing::{process_operations, verify_exit::verify_exit}, BlockSignatureStrategy, ConsensusContext, VerifyBlockRoot, VerifySignatures, @@ -72,7 +72,6 @@ async fn valid_block_ok() { &mut state, &block, BlockSignatureStrategy::VerifyIndividual, - StateProcessingStrategy::Accurate, VerifyBlockRoot::True, &mut ctxt, &spec, @@ -98,7 +97,6 @@ async fn invalid_block_header_state_slot() { &mut state, &SignedBeaconBlock::from_block(block, signature), BlockSignatureStrategy::VerifyIndividual, - StateProcessingStrategy::Accurate, VerifyBlockRoot::True, &mut ctxt, &spec, @@ -131,7 +129,6 @@ async fn invalid_parent_block_root() { &mut state, &SignedBeaconBlock::from_block(block, signature), BlockSignatureStrategy::VerifyIndividual, - StateProcessingStrategy::Accurate, VerifyBlockRoot::True, &mut ctxt, &spec, @@ -165,7 +162,6 @@ async fn invalid_block_signature() { &mut state, &SignedBeaconBlock::from_block(block, Signature::empty()), BlockSignatureStrategy::VerifyIndividual, - StateProcessingStrategy::Accurate, VerifyBlockRoot::True, &mut ctxt, &spec, @@ -199,7 +195,6 @@ async fn invalid_randao_reveal_signature() { &mut state, &signed_block, BlockSignatureStrategy::VerifyIndividual, - StateProcessingStrategy::Accurate, VerifyBlockRoot::True, &mut ctxt, &spec, diff --git a/consensus/state_processing/src/per_epoch_processing/altair/participation_flag_updates.rs b/consensus/state_processing/src/per_epoch_processing/altair/participation_flag_updates.rs index dd1b2dfcd86..fc55fb11144 100644 --- a/consensus/state_processing/src/per_epoch_processing/altair/participation_flag_updates.rs +++ b/consensus/state_processing/src/per_epoch_processing/altair/participation_flag_updates.rs @@ -2,17 +2,14 @@ use crate::EpochProcessingError; use types::beacon_state::BeaconState; use types::eth_spec::EthSpec; use types::participation_flags::ParticipationFlags; -use types::VariableList; +use types::List; pub fn process_participation_flag_updates( state: &mut BeaconState, ) -> Result<(), EpochProcessingError> { *state.previous_epoch_participation_mut()? = std::mem::take(state.current_epoch_participation_mut()?); - *state.current_epoch_participation_mut()? = VariableList::new(vec![ - ParticipationFlags::default( - ); - state.validators().len() - ])?; + *state.current_epoch_participation_mut()? = + List::repeat(ParticipationFlags::default(), state.validators().len())?; Ok(()) } diff --git a/consensus/state_processing/src/per_epoch_processing/capella/historical_summaries_update.rs b/consensus/state_processing/src/per_epoch_processing/capella/historical_summaries_update.rs index 7490f276567..00adabdcfe9 100644 --- a/consensus/state_processing/src/per_epoch_processing/capella/historical_summaries_update.rs +++ b/consensus/state_processing/src/per_epoch_processing/capella/historical_summaries_update.rs @@ -13,6 +13,9 @@ pub fn process_historical_summaries_update( .safe_rem((E::slots_per_historical_root() as u64).safe_div(E::slots_per_epoch())?)? == 0 { + // We need to flush any pending mutations before hashing. + state.block_roots_mut().apply_updates()?; + state.state_roots_mut().apply_updates()?; let summary = HistoricalSummary::new(state); return state .historical_summaries_mut()? diff --git a/consensus/state_processing/src/per_epoch_processing/effective_balance_updates.rs b/consensus/state_processing/src/per_epoch_processing/effective_balance_updates.rs index 7bd62c40816..73881e932b7 100644 --- a/consensus/state_processing/src/per_epoch_processing/effective_balance_updates.rs +++ b/consensus/state_processing/src/per_epoch_processing/effective_balance_updates.rs @@ -21,7 +21,9 @@ pub fn process_effective_balance_updates( let downward_threshold = hysteresis_increment.safe_mul(spec.hysteresis_downward_multiplier)?; let upward_threshold = hysteresis_increment.safe_mul(spec.hysteresis_upward_multiplier)?; let (validators, balances, _) = state.validators_and_balances_and_progressive_balances_mut(); - for (index, validator) in validators.iter_mut().enumerate() { + let mut validators_iter = validators.iter_cow(); + + while let Some((index, validator)) = validators_iter.next_cow() { let balance = balances .get(index) .copied() @@ -44,7 +46,7 @@ pub fn process_effective_balance_updates( } if new_effective_balance != validator.effective_balance { - validator.effective_balance = new_effective_balance; + validator.into_mut()?.effective_balance = new_effective_balance; } } diff --git a/consensus/state_processing/src/per_epoch_processing/epoch_processing_summary.rs b/consensus/state_processing/src/per_epoch_processing/epoch_processing_summary.rs index 65a946e7bff..6f48050e161 100644 --- a/consensus/state_processing/src/per_epoch_processing/epoch_processing_summary.rs +++ b/consensus/state_processing/src/per_epoch_processing/epoch_processing_summary.rs @@ -3,8 +3,8 @@ use crate::metrics; use std::sync::Arc; use types::{ consts::altair::{TIMELY_HEAD_FLAG_INDEX, TIMELY_SOURCE_FLAG_INDEX, TIMELY_TARGET_FLAG_INDEX}, - BeaconStateError, Epoch, EthSpec, ParticipationFlags, ProgressiveBalancesCache, SyncCommittee, - Validator, VariableList, + BeaconStateError, Epoch, EthSpec, List, ParticipationFlags, ProgressiveBalancesCache, + SyncCommittee, Validator, }; /// Provides a summary of validator participation during the epoch. @@ -25,20 +25,20 @@ pub enum EpochProcessingSummary { #[derive(PartialEq, Debug)] pub struct ParticipationEpochSummary { /// Copy of the validator registry prior to mutation. - validators: VariableList, + validators: List, /// Copy of the participation flags for the previous epoch. - previous_epoch_participation: VariableList, + previous_epoch_participation: List, /// Copy of the participation flags for the current epoch. - current_epoch_participation: VariableList, + current_epoch_participation: List, previous_epoch: Epoch, current_epoch: Epoch, } impl ParticipationEpochSummary { pub fn new( - validators: VariableList, - previous_epoch_participation: VariableList, - current_epoch_participation: VariableList, + validators: List, + previous_epoch_participation: List, + current_epoch_participation: List, previous_epoch: Epoch, current_epoch: Epoch, ) -> Self { diff --git a/consensus/state_processing/src/per_epoch_processing/errors.rs b/consensus/state_processing/src/per_epoch_processing/errors.rs index c18e1303b26..de481ec6767 100644 --- a/consensus/state_processing/src/per_epoch_processing/errors.rs +++ b/consensus/state_processing/src/per_epoch_processing/errors.rs @@ -1,4 +1,4 @@ -use types::{BeaconStateError, EpochCacheError, InconsistentFork}; +use types::{milhouse, BeaconStateError, EpochCacheError, InconsistentFork}; #[derive(Debug, PartialEq)] pub enum EpochProcessingError { @@ -23,6 +23,7 @@ pub enum EpochProcessingError { InconsistentStateFork(InconsistentFork), InvalidJustificationBit(ssz_types::Error), InvalidFlagIndex(usize), + MilhouseError(milhouse::Error), EpochCache(EpochCacheError), } @@ -50,6 +51,12 @@ impl From for EpochProcessingError { } } +impl From for EpochProcessingError { + fn from(e: milhouse::Error) -> Self { + Self::MilhouseError(e) + } +} + impl From for EpochProcessingError { fn from(e: EpochCacheError) -> Self { EpochProcessingError::EpochCache(e) diff --git a/consensus/state_processing/src/per_epoch_processing/historical_roots_update.rs b/consensus/state_processing/src/per_epoch_processing/historical_roots_update.rs index 6d06b4d7ca5..7686932192f 100644 --- a/consensus/state_processing/src/per_epoch_processing/historical_roots_update.rs +++ b/consensus/state_processing/src/per_epoch_processing/historical_roots_update.rs @@ -14,7 +14,7 @@ pub fn process_historical_roots_update( .safe_rem(E::SlotsPerHistoricalRoot::to_u64().safe_div(E::slots_per_epoch())?)? == 0 { - let historical_batch = state.historical_batch(); + let historical_batch = state.historical_batch()?; state .historical_roots_mut() .push(historical_batch.tree_hash_root())?; diff --git a/consensus/state_processing/src/per_epoch_processing/resets.rs b/consensus/state_processing/src/per_epoch_processing/resets.rs index d577c52e6a5..c9f69c3c95e 100644 --- a/consensus/state_processing/src/per_epoch_processing/resets.rs +++ b/consensus/state_processing/src/per_epoch_processing/resets.rs @@ -2,7 +2,7 @@ use super::errors::EpochProcessingError; use safe_arith::SafeArith; use types::beacon_state::BeaconState; use types::eth_spec::EthSpec; -use types::{Unsigned, VariableList}; +use types::{List, Unsigned}; pub fn process_eth1_data_reset( state: &mut BeaconState, @@ -13,7 +13,7 @@ pub fn process_eth1_data_reset( .safe_rem(E::SlotsPerEth1VotingPeriod::to_u64())? == 0 { - *state.eth1_data_votes_mut() = VariableList::empty(); + *state.eth1_data_votes_mut() = List::empty(); } Ok(()) } diff --git a/consensus/state_processing/src/per_epoch_processing/single_pass.rs b/consensus/state_processing/src/per_epoch_processing/single_pass.rs index 380484046c3..7a95de3317e 100644 --- a/consensus/state_processing/src/per_epoch_processing/single_pass.rs +++ b/consensus/state_processing/src/per_epoch_processing/single_pass.rs @@ -12,6 +12,7 @@ use types::{ NUM_FLAG_INDICES, PARTICIPATION_FLAG_WEIGHTS, TIMELY_HEAD_FLAG_INDEX, TIMELY_TARGET_FLAG_INDEX, WEIGHT_DENOMINATOR, }, + milhouse::Cow, ActivationQueue, BeaconState, BeaconStateError, ChainSpec, Epoch, EthSpec, ExitCache, ForkName, ParticipationFlags, ProgressiveBalancesCache, RelativeEpoch, Unsigned, Validator, }; @@ -173,9 +174,9 @@ pub fn process_epoch_single_pass( let effective_balances_ctxt = &EffectiveBalancesContext::new(spec)?; // Iterate over the validators and related fields in one pass. - let mut validators_iter = validators.iter_mut(); - let mut balances_iter = balances.iter_mut(); - let mut inactivity_scores_iter = inactivity_scores.iter_mut(); + let mut validators_iter = validators.iter_cow(); + let mut balances_iter = balances.iter_cow(); + let mut inactivity_scores_iter = inactivity_scores.iter_cow(); // Values computed for the next epoch transition. let mut next_epoch_total_active_balance = 0; @@ -186,14 +187,14 @@ pub fn process_epoch_single_pass( previous_epoch_participation.iter(), current_epoch_participation.iter(), ) { - let validator = validators_iter - .next() + let (_, mut validator) = validators_iter + .next_cow() .ok_or(BeaconStateError::UnknownValidator(index))?; - let balance = balances_iter - .next() + let (_, mut balance) = balances_iter + .next_cow() .ok_or(BeaconStateError::UnknownValidator(index))?; - let inactivity_score = inactivity_scores_iter - .next() + let (_, mut inactivity_score) = inactivity_scores_iter + .next_cow() .ok_or(BeaconStateError::UnknownValidator(index))?; let is_active_current_epoch = validator.is_active_at(current_epoch); @@ -223,7 +224,7 @@ pub fn process_epoch_single_pass( // `process_inactivity_updates` if conf.inactivity_updates { process_single_inactivity_update( - inactivity_score, + &mut inactivity_score, validator_info, state_ctxt, spec, @@ -233,8 +234,8 @@ pub fn process_epoch_single_pass( // `process_rewards_and_penalties` if conf.rewards_and_penalties { process_single_reward_and_penalty( - balance, - inactivity_score, + &mut balance, + &inactivity_score, validator_info, rewards_ctxt, state_ctxt, @@ -246,7 +247,7 @@ pub fn process_epoch_single_pass( // `process_registry_updates` if conf.registry_updates { process_single_registry_update( - validator, + &mut validator, validator_info, exit_cache, activation_queue, @@ -258,14 +259,14 @@ pub fn process_epoch_single_pass( // `process_slashings` if conf.slashings { - process_single_slashing(balance, validator, slashings_ctxt, state_ctxt, spec)?; + process_single_slashing(&mut balance, &validator, slashings_ctxt, state_ctxt, spec)?; } // `process_effective_balance_updates` if conf.effective_balance_updates { process_single_effective_balance_update( *balance, - validator, + &mut validator, validator_info, &mut next_epoch_total_active_balance, &mut next_epoch_cache, @@ -290,7 +291,7 @@ pub fn process_epoch_single_pass( } fn process_single_inactivity_update( - inactivity_score: &mut u64, + inactivity_score: &mut Cow, validator_info: &ValidatorInfo, state_ctxt: &StateContext, spec: &ChainSpec, @@ -303,25 +304,27 @@ fn process_single_inactivity_update( if validator_info.is_unslashed_participating_index(TIMELY_TARGET_FLAG_INDEX)? { // Avoid mutating when the inactivity score is 0 and can't go any lower -- the common // case. - if *inactivity_score == 0 { + if **inactivity_score == 0 { return Ok(()); } - inactivity_score.safe_sub_assign(1)?; + inactivity_score.make_mut()?.safe_sub_assign(1)?; } else { - inactivity_score.safe_add_assign(spec.inactivity_score_bias)?; + inactivity_score + .make_mut()? + .safe_add_assign(spec.inactivity_score_bias)?; } // Decrease the score of all validators for forgiveness when not during a leak if !state_ctxt.is_in_inactivity_leak { - let deduction = min(spec.inactivity_score_recovery_rate, *inactivity_score); - inactivity_score.safe_sub_assign(deduction)?; + let deduction = min(spec.inactivity_score_recovery_rate, **inactivity_score); + inactivity_score.make_mut()?.safe_sub_assign(deduction)?; } Ok(()) } fn process_single_reward_and_penalty( - balance: &mut u64, + balance: &mut Cow, inactivity_score: &u64, validator_info: &ValidatorInfo, rewards_ctxt: &RewardsAndPenaltiesContext, @@ -351,6 +354,7 @@ fn process_single_reward_and_penalty( )?; if delta.rewards != 0 || delta.penalties != 0 { + let balance = balance.make_mut()?; balance.safe_add_assign(delta.rewards)?; *balance = balance.saturating_sub(delta.penalties); } @@ -452,7 +456,7 @@ impl RewardsAndPenaltiesContext { } fn process_single_registry_update( - validator: &mut Validator, + validator: &mut Cow, validator_info: &ValidatorInfo, exit_cache: &mut ExitCache, activation_queue: &BTreeSet, @@ -463,7 +467,7 @@ fn process_single_registry_update( let current_epoch = state_ctxt.current_epoch; if validator.is_eligible_for_activation_queue(spec) { - validator.activation_eligibility_epoch = current_epoch.safe_add(1)?; + validator.make_mut()?.activation_eligibility_epoch = current_epoch.safe_add(1)?; } if validator.is_active_at(current_epoch) && validator.effective_balance <= spec.ejection_balance @@ -472,7 +476,8 @@ fn process_single_registry_update( } if activation_queue.contains(&validator_info.index) { - validator.activation_epoch = spec.compute_activation_exit_epoch(current_epoch)?; + validator.make_mut()?.activation_epoch = + spec.compute_activation_exit_epoch(current_epoch)?; } // Caching: add to speculative activation queue for next epoch. @@ -487,7 +492,7 @@ fn process_single_registry_update( } fn initiate_validator_exit( - validator: &mut Validator, + validator: &mut Cow, exit_cache: &mut ExitCache, state_ctxt: &StateContext, spec: &ChainSpec, @@ -508,6 +513,7 @@ fn initiate_validator_exit( exit_queue_epoch.safe_add_assign(1)?; } + let validator = validator.make_mut()?; validator.exit_epoch = exit_queue_epoch; validator.withdrawable_epoch = exit_queue_epoch.safe_add(spec.min_validator_withdrawability_delay)?; @@ -540,7 +546,7 @@ impl SlashingsContext { } fn process_single_slashing( - balance: &mut u64, + balance: &mut Cow, validator: &Validator, slashings_ctxt: &SlashingsContext, state_ctxt: &StateContext, @@ -557,7 +563,7 @@ fn process_single_slashing( .safe_div(state_ctxt.total_active_balance)? .safe_mul(increment)?; - *balance = balance.saturating_sub(penalty); + *balance.make_mut()? = balance.saturating_sub(penalty); } Ok(()) } @@ -581,7 +587,7 @@ impl EffectiveBalancesContext { #[allow(clippy::too_many_arguments)] fn process_single_effective_balance_update( balance: u64, - validator: &mut Validator, + validator: &mut Cow, validator_info: &ValidatorInfo, next_epoch_total_active_balance: &mut u64, next_epoch_cache: &mut PreEpochCache, @@ -611,7 +617,7 @@ fn process_single_effective_balance_update( } if new_effective_balance != old_effective_balance { - validator.effective_balance = new_effective_balance; + validator.make_mut()?.effective_balance = new_effective_balance; // Update progressive balances cache for the *current* epoch, which will soon become the // previous epoch once the epoch transition completes. diff --git a/consensus/state_processing/src/upgrade/altair.rs b/consensus/state_processing/src/upgrade/altair.rs index cfbc6eba9e9..872560db3df 100644 --- a/consensus/state_processing/src/upgrade/altair.rs +++ b/consensus/state_processing/src/upgrade/altair.rs @@ -4,13 +4,13 @@ use std::mem; use std::sync::Arc; use types::{ BeaconState, BeaconStateAltair, BeaconStateError as Error, ChainSpec, EpochCache, EthSpec, - Fork, ParticipationFlags, PendingAttestation, RelativeEpoch, SyncCommittee, VariableList, + Fork, List, ParticipationFlags, PendingAttestation, RelativeEpoch, SyncCommittee, }; /// Translate the participation information from the epoch prior to the fork into Altair's format. pub fn translate_participation( state: &mut BeaconState, - pending_attestations: &VariableList, E::MaxPendingAttestations>, + pending_attestations: &List, E::MaxPendingAttestations>, spec: &ChainSpec, ) -> Result<(), Error> { // Previous epoch committee cache is required for `get_attesting_indices`. @@ -51,8 +51,8 @@ pub fn upgrade_to_altair( let pre = pre_state.as_base_mut()?; let default_epoch_participation = - VariableList::new(vec![ParticipationFlags::default(); pre.validators.len()])?; - let inactivity_scores = VariableList::new(vec![0; pre.validators.len()])?; + List::new(vec![ParticipationFlags::default(); pre.validators.len()])?; + let inactivity_scores = List::new(vec![0; pre.validators.len()])?; let temp_sync_committee = Arc::new(SyncCommittee::temporary()); @@ -108,7 +108,6 @@ pub fn upgrade_to_altair( exit_cache: mem::take(&mut pre.exit_cache), slashings_cache: mem::take(&mut pre.slashings_cache), epoch_cache: EpochCache::default(), - tree_hash_cache: mem::take(&mut pre.tree_hash_cache), }); // Fill in previous epoch participation from the pre state's pending attestations. diff --git a/consensus/state_processing/src/upgrade/capella.rs b/consensus/state_processing/src/upgrade/capella.rs index 87b40abebdd..51e29d10f3c 100644 --- a/consensus/state_processing/src/upgrade/capella.rs +++ b/consensus/state_processing/src/upgrade/capella.rs @@ -1,7 +1,7 @@ use std::mem; use types::{ BeaconState, BeaconStateCapella, BeaconStateError as Error, ChainSpec, EpochCache, EthSpec, - Fork, VariableList, + Fork, List, }; /// Transform a `Merge` state into an `Capella` state. @@ -61,7 +61,7 @@ pub fn upgrade_to_capella( // Capella next_withdrawal_index: 0, next_withdrawal_validator_index: 0, - historical_summaries: VariableList::default(), + historical_summaries: List::default(), // Caches total_active_balance: pre.total_active_balance, progressive_balances_cache: mem::take(&mut pre.progressive_balances_cache), @@ -70,7 +70,6 @@ pub fn upgrade_to_capella( exit_cache: mem::take(&mut pre.exit_cache), slashings_cache: mem::take(&mut pre.slashings_cache), epoch_cache: EpochCache::default(), - tree_hash_cache: mem::take(&mut pre.tree_hash_cache), }); *pre_state = post; diff --git a/consensus/state_processing/src/upgrade/deneb.rs b/consensus/state_processing/src/upgrade/deneb.rs index 43fe5d9dc3d..c21e1361a5a 100644 --- a/consensus/state_processing/src/upgrade/deneb.rs +++ b/consensus/state_processing/src/upgrade/deneb.rs @@ -71,7 +71,6 @@ pub fn upgrade_to_deneb( exit_cache: mem::take(&mut pre.exit_cache), slashings_cache: mem::take(&mut pre.slashings_cache), epoch_cache: EpochCache::default(), - tree_hash_cache: mem::take(&mut pre.tree_hash_cache), }); *pre_state = post; diff --git a/consensus/state_processing/src/upgrade/electra.rs b/consensus/state_processing/src/upgrade/electra.rs index a37d0fc3beb..f64228f050b 100644 --- a/consensus/state_processing/src/upgrade/electra.rs +++ b/consensus/state_processing/src/upgrade/electra.rs @@ -70,7 +70,6 @@ pub fn upgrade_to_electra( exit_cache: mem::take(&mut pre.exit_cache), slashings_cache: mem::take(&mut pre.slashings_cache), epoch_cache: EpochCache::default(), - tree_hash_cache: mem::take(&mut pre.tree_hash_cache), }); *pre_state = post; diff --git a/consensus/state_processing/src/upgrade/merge.rs b/consensus/state_processing/src/upgrade/merge.rs index 147c97ac29e..02705743ceb 100644 --- a/consensus/state_processing/src/upgrade/merge.rs +++ b/consensus/state_processing/src/upgrade/merge.rs @@ -66,7 +66,6 @@ pub fn upgrade_to_bellatrix( exit_cache: mem::take(&mut pre.exit_cache), slashings_cache: mem::take(&mut pre.slashings_cache), epoch_cache: EpochCache::default(), - tree_hash_cache: mem::take(&mut pre.tree_hash_cache), }); *pre_state = post; diff --git a/consensus/types/Cargo.toml b/consensus/types/Cargo.toml index db15f53537e..4b7d9f2b98d 100644 --- a/consensus/types/Cargo.toml +++ b/consensus/types/Cargo.toml @@ -52,6 +52,8 @@ serde_json = { workspace = true } smallvec = { workspace = true } maplit = { workspace = true } strum = { workspace = true } +milhouse = { workspace = true } +rpds = { workspace = true } [dev-dependencies] criterion = { workspace = true } @@ -68,4 +70,4 @@ sqlite = [] # The `arbitrary-fuzz` feature is a no-op provided for backwards compatibility. # For simplicity `Arbitrary` is now derived regardless of the feature's presence. arbitrary-fuzz = [] -portable = ["bls/supranational-portable"] \ No newline at end of file +portable = ["bls/supranational-portable"] diff --git a/consensus/types/benches/benches.rs b/consensus/types/benches/benches.rs index bb2b527109f..5c1036a4c5a 100644 --- a/consensus/types/benches/benches.rs +++ b/consensus/types/benches/benches.rs @@ -2,6 +2,7 @@ use criterion::Criterion; use criterion::{black_box, criterion_group, criterion_main, Benchmark}; +use milhouse::List; use rayon::prelude::*; use ssz::Encode; use std::sync::Arc; @@ -27,21 +28,23 @@ fn get_state(validator_count: usize) -> BeaconState { .expect("should add balance"); } - *state.validators_mut() = (0..validator_count) - .collect::>() - .par_iter() - .map(|&i| Validator { - pubkey: generate_deterministic_keypair(i).pk.into(), - withdrawal_credentials: Hash256::from_low_u64_le(i as u64), - effective_balance: spec.max_effective_balance, - slashed: false, - activation_eligibility_epoch: Epoch::new(0), - activation_epoch: Epoch::new(0), - exit_epoch: Epoch::from(u64::max_value()), - withdrawable_epoch: Epoch::from(u64::max_value()), - }) - .collect::>() - .into(); + *state.validators_mut() = List::new( + (0..validator_count) + .collect::>() + .par_iter() + .map(|&i| Validator { + pubkey: generate_deterministic_keypair(i).pk.compress(), + withdrawal_credentials: Hash256::from_low_u64_le(i as u64), + effective_balance: spec.max_effective_balance, + slashed: false, + activation_eligibility_epoch: Epoch::new(0), + activation_epoch: Epoch::new(0), + exit_epoch: Epoch::from(u64::max_value()), + withdrawable_epoch: Epoch::from(u64::max_value()), + }) + .collect(), + ) + .unwrap(); state } @@ -96,19 +99,6 @@ fn all_benches(c: &mut Criterion) { .sample_size(10), ); - let inner_state = state.clone(); - c.bench( - &format!("{}_validators", validator_count), - Benchmark::new("clone/tree_hash_cache", move |b| { - b.iter_batched_ref( - || inner_state.clone(), - |state| black_box(state.tree_hash_cache().clone()), - criterion::BatchSize::SmallInput, - ) - }) - .sample_size(10), - ); - let inner_state = state.clone(); c.bench( &format!("{}_validators", validator_count), diff --git a/consensus/types/examples/clone_state.rs b/consensus/types/examples/clone_state.rs deleted file mode 100644 index a7e80cf4078..00000000000 --- a/consensus/types/examples/clone_state.rs +++ /dev/null @@ -1,51 +0,0 @@ -//! These examples only really exist so we can use them for flamegraph. If they get annoying to -//! maintain, feel free to delete. - -use types::{ - test_utils::generate_deterministic_keypair, BeaconState, Eth1Data, EthSpec, Hash256, - MinimalEthSpec, Validator, -}; - -type E = MinimalEthSpec; - -fn get_state(validator_count: usize) -> BeaconState { - let spec = &E::default_spec(); - let eth1_data = Eth1Data { - deposit_root: Hash256::zero(), - deposit_count: 0, - block_hash: Hash256::zero(), - }; - - let mut state = BeaconState::new(0, eth1_data, spec); - - for i in 0..validator_count { - state - .balances_mut() - .push(i as u64) - .expect("should add balance"); - state - .validators_mut() - .push(Validator { - pubkey: generate_deterministic_keypair(i).pk.into(), - withdrawal_credentials: Hash256::from_low_u64_le(i as u64), - effective_balance: i as u64, - slashed: i % 2 == 0, - activation_eligibility_epoch: i.into(), - activation_epoch: i.into(), - exit_epoch: i.into(), - withdrawable_epoch: i.into(), - }) - .expect("should add validator"); - } - - state -} - -fn main() { - let validator_count = 1_024; - let state = get_state(validator_count); - - for _ in 0..100_000 { - let _ = state.clone(); - } -} diff --git a/consensus/types/examples/ssz_encode_state.rs b/consensus/types/examples/ssz_encode_state.rs deleted file mode 100644 index 5d0a2db17c7..00000000000 --- a/consensus/types/examples/ssz_encode_state.rs +++ /dev/null @@ -1,54 +0,0 @@ -//! These examples only really exist so we can use them for flamegraph. If they get annoying to -//! maintain, feel free to delete. - -use ssz::Encode; -use types::{ - test_utils::generate_deterministic_keypair, BeaconState, Eth1Data, EthSpec, Hash256, - MinimalEthSpec, Validator, -}; - -type E = MinimalEthSpec; - -fn get_state(validator_count: usize) -> BeaconState { - let spec = &E::default_spec(); - let eth1_data = Eth1Data { - deposit_root: Hash256::zero(), - deposit_count: 0, - block_hash: Hash256::zero(), - }; - - let mut state = BeaconState::new(0, eth1_data, spec); - - for i in 0..validator_count { - state - .balances_mut() - .push(i as u64) - .expect("should add balance"); - state - .validators_mut() - .push(Validator { - pubkey: generate_deterministic_keypair(i).pk.into(), - withdrawal_credentials: Hash256::from_low_u64_le(i as u64), - effective_balance: i as u64, - slashed: i % 2 == 0, - activation_eligibility_epoch: i.into(), - activation_epoch: i.into(), - exit_epoch: i.into(), - withdrawable_epoch: i.into(), - }) - .expect("should add validator"); - } - - state -} - -fn main() { - let validator_count = 1_024; - let state = get_state(validator_count); - - for _ in 0..1_024 { - let state_bytes = state.as_ssz_bytes(); - let _: BeaconState = - BeaconState::from_ssz_bytes(&state_bytes, &E::default_spec()).expect("should decode"); - } -} diff --git a/consensus/types/examples/tree_hash_state.rs b/consensus/types/examples/tree_hash_state.rs deleted file mode 100644 index 26777b25912..00000000000 --- a/consensus/types/examples/tree_hash_state.rs +++ /dev/null @@ -1,56 +0,0 @@ -//! These examples only really exist so we can use them for flamegraph. If they get annoying to -//! maintain, feel free to delete. - -use types::{ - test_utils::generate_deterministic_keypair, BeaconState, Eth1Data, EthSpec, Hash256, - MinimalEthSpec, Validator, -}; - -type E = MinimalEthSpec; - -fn get_state(validator_count: usize) -> BeaconState { - let spec = &E::default_spec(); - let eth1_data = Eth1Data { - deposit_root: Hash256::zero(), - deposit_count: 0, - block_hash: Hash256::zero(), - }; - - let mut state = BeaconState::new(0, eth1_data, spec); - - for i in 0..validator_count { - state - .balances_mut() - .push(i as u64) - .expect("should add balance"); - state - .validators_mut() - .push(Validator { - pubkey: generate_deterministic_keypair(i).pk.into(), - withdrawal_credentials: Hash256::from_low_u64_le(i as u64), - effective_balance: i as u64, - slashed: i % 2 == 0, - activation_eligibility_epoch: i.into(), - activation_epoch: i.into(), - exit_epoch: i.into(), - withdrawable_epoch: i.into(), - }) - .expect("should add validator"); - } - - state -} - -fn main() { - let validator_count = 1_024; - let mut state = get_state(validator_count); - state.update_tree_hash_cache().expect("should update cache"); - - actual_thing::(&mut state); -} - -fn actual_thing(state: &mut BeaconState) { - for _ in 0..200_024 { - let _ = state.update_tree_hash_cache().expect("should update cache"); - } -} diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 02572b0efbd..5da81f6a752 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -7,6 +7,7 @@ use compare_fields_derive::CompareFields; use derivative::Derivative; use ethereum_hashing::hash; use int_to_bytes::{int_to_bytes4, int_to_bytes8}; +use metastruct::{metastruct, NumFields}; pub use pubkey_cache::PubkeyCache; use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; @@ -28,28 +29,25 @@ pub use crate::beacon_state::balance::Balance; pub use crate::beacon_state::exit_cache::ExitCache; pub use crate::beacon_state::progressive_balances_cache::*; pub use crate::beacon_state::slashings_cache::SlashingsCache; -pub use clone_config::CloneConfig; pub use eth_spec::*; pub use iter::BlockRootsIter; -pub use tree_hash_cache::BeaconTreeHashCache; +pub use milhouse::{interface::Interface, List, Vector}; #[macro_use] mod committee_cache; mod balance; -mod clone_config; mod exit_cache; mod iter; mod progressive_balances_cache; mod pubkey_cache; mod slashings_cache; mod tests; -mod tree_hash_cache; pub const CACHED_EPOCHS: usize = 3; const MAX_RANDOM_BYTE: u64 = (1 << 8) - 1; -pub type Validators = VariableList::ValidatorRegistryLimit>; -pub type Balances = VariableList::ValidatorRegistryLimit>; +pub type Validators = List::ValidatorRegistryLimit>; +pub type Balances = List::ValidatorRegistryLimit>; #[derive(Debug, PartialEq, Clone)] pub enum Error { @@ -144,6 +142,20 @@ pub enum Error { current_epoch: Epoch, epoch: Epoch, }, + MilhouseError(milhouse::Error), + CommitteeCacheDiffInvalidEpoch { + prev_current_epoch: Epoch, + current_epoch: Epoch, + }, + CommitteeCacheDiffUninitialized { + expected_epoch: Epoch, + }, + DiffAcrossFork { + prev_fork: ForkName, + current_fork: ForkName, + }, + TotalActiveBalanceDiffUninitialized, + MissingImmutableValidator(usize), IndexNotSupported(usize), InvalidFlagIndex(usize), MerkleTreeError(merkle_proof::MerkleTreeError), @@ -207,16 +219,105 @@ impl From for Hash256 { TreeHash, TestRandom, CompareFields, - arbitrary::Arbitrary + arbitrary::Arbitrary, ), serde(bound = "E: EthSpec", deny_unknown_fields), arbitrary(bound = "E: EthSpec"), derivative(Clone), ), + specific_variant_attributes( + Base(metastruct( + mappings( + map_beacon_state_base_fields(), + map_beacon_state_base_tree_list_fields(mutable, fallible, groups(tree_lists)), + map_beacon_state_base_tree_list_fields_immutable(groups(tree_lists)), + ), + bimappings(bimap_beacon_state_base_tree_list_fields( + other_type = "BeaconStateBase", + self_mutable, + fallible, + groups(tree_lists) + )), + num_fields(all()), + )), + Altair(metastruct( + mappings( + map_beacon_state_altair_fields(), + map_beacon_state_altair_tree_list_fields(mutable, fallible, groups(tree_lists)), + map_beacon_state_altair_tree_list_fields_immutable(groups(tree_lists)), + ), + bimappings(bimap_beacon_state_altair_tree_list_fields( + other_type = "BeaconStateAltair", + self_mutable, + fallible, + groups(tree_lists) + )), + num_fields(all()), + )), + Merge(metastruct( + mappings( + map_beacon_state_bellatrix_fields(), + map_beacon_state_bellatrix_tree_list_fields(mutable, fallible, groups(tree_lists)), + map_beacon_state_bellatrix_tree_list_fields_immutable(groups(tree_lists)), + ), + bimappings(bimap_beacon_state_bellatrix_tree_list_fields( + other_type = "BeaconStateMerge", + self_mutable, + fallible, + groups(tree_lists) + )), + num_fields(all()), + )), + Capella(metastruct( + mappings( + map_beacon_state_capella_fields(), + map_beacon_state_capella_tree_list_fields(mutable, fallible, groups(tree_lists)), + map_beacon_state_capella_tree_list_fields_immutable(groups(tree_lists)), + ), + bimappings(bimap_beacon_state_capella_tree_list_fields( + other_type = "BeaconStateCapella", + self_mutable, + fallible, + groups(tree_lists) + )), + num_fields(all()), + )), + Deneb(metastruct( + mappings( + map_beacon_state_deneb_fields(), + map_beacon_state_deneb_tree_list_fields(mutable, fallible, groups(tree_lists)), + map_beacon_state_deneb_tree_list_fields_immutable(groups(tree_lists)), + ), + bimappings(bimap_beacon_state_deneb_tree_list_fields( + other_type = "BeaconStateDeneb", + self_mutable, + fallible, + groups(tree_lists) + )), + num_fields(all()), + )), + Electra(metastruct( + mappings( + map_beacon_state_electra_fields(), + map_beacon_state_electra_tree_list_fields(mutable, fallible, groups(tree_lists)), + map_beacon_state_electra_tree_list_fields_immutable(groups(tree_lists)), + ), + bimappings(bimap_beacon_state_electra_tree_list_fields( + other_type = "BeaconStateElectra", + self_mutable, + fallible, + groups(tree_lists) + )), + num_fields(all()), + )) + ), cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"), - partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant") + partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant"), + map_ref_mut_into(BeaconStateRef) +)] +#[derive( + Debug, PartialEq, Clone, Serialize, Deserialize, Encode, TreeHash, arbitrary::Arbitrary, )] -#[derive(Debug, PartialEq, Serialize, Deserialize, Encode, TreeHash, arbitrary::Arbitrary)] #[serde(untagged)] #[serde(bound = "E: EthSpec")] #[arbitrary(bound = "E: EthSpec")] @@ -228,76 +329,102 @@ where { // Versioning #[superstruct(getter(copy))] + #[metastruct(exclude_from(tree_lists))] #[serde(with = "serde_utils::quoted_u64")] pub genesis_time: u64, #[superstruct(getter(copy))] + #[metastruct(exclude_from(tree_lists))] pub genesis_validators_root: Hash256, #[superstruct(getter(copy))] + #[metastruct(exclude_from(tree_lists))] pub slot: Slot, #[superstruct(getter(copy))] + #[metastruct(exclude_from(tree_lists))] pub fork: Fork, // History + #[metastruct(exclude_from(tree_lists))] pub latest_block_header: BeaconBlockHeader, - #[compare_fields(as_slice)] - pub block_roots: FixedVector, - #[compare_fields(as_slice)] - pub state_roots: FixedVector, + #[test_random(default)] + #[compare_fields(as_iter)] + pub block_roots: Vector, + #[test_random(default)] + #[compare_fields(as_iter)] + pub state_roots: Vector, // Frozen in Capella, replaced by historical_summaries - pub historical_roots: VariableList, + #[test_random(default)] + #[compare_fields(as_iter)] + pub historical_roots: List, // Ethereum 1.0 chain data + #[metastruct(exclude_from(tree_lists))] pub eth1_data: Eth1Data, - pub eth1_data_votes: VariableList, + #[test_random(default)] + pub eth1_data_votes: List, #[superstruct(getter(copy))] + #[metastruct(exclude_from(tree_lists))] #[serde(with = "serde_utils::quoted_u64")] pub eth1_deposit_index: u64, // Registry - #[compare_fields(as_slice)] - pub validators: VariableList, - #[compare_fields(as_slice)] + #[test_random(default)] + pub validators: List, #[serde(with = "ssz_types::serde_utils::quoted_u64_var_list")] - pub balances: VariableList, + #[compare_fields(as_iter)] + #[test_random(default)] + pub balances: List, // Randomness - pub randao_mixes: FixedVector, + #[test_random(default)] + pub randao_mixes: Vector, // Slashings + #[test_random(default)] #[serde(with = "ssz_types::serde_utils::quoted_u64_fixed_vec")] - pub slashings: FixedVector, + pub slashings: Vector, // Attestations (genesis fork only) #[superstruct(only(Base))] - pub previous_epoch_attestations: VariableList, E::MaxPendingAttestations>, + #[test_random(default)] + pub previous_epoch_attestations: List, E::MaxPendingAttestations>, #[superstruct(only(Base))] - pub current_epoch_attestations: VariableList, E::MaxPendingAttestations>, + #[test_random(default)] + pub current_epoch_attestations: List, E::MaxPendingAttestations>, // Participation (Altair and later) #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] - pub previous_epoch_participation: VariableList, + #[test_random(default)] + pub previous_epoch_participation: List, #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] - pub current_epoch_participation: VariableList, + #[test_random(default)] + pub current_epoch_participation: List, // Finality #[test_random(default)] + #[metastruct(exclude_from(tree_lists))] pub justification_bits: BitVector, #[superstruct(getter(copy))] + #[metastruct(exclude_from(tree_lists))] pub previous_justified_checkpoint: Checkpoint, #[superstruct(getter(copy))] + #[metastruct(exclude_from(tree_lists))] pub current_justified_checkpoint: Checkpoint, #[superstruct(getter(copy))] + #[metastruct(exclude_from(tree_lists))] pub finalized_checkpoint: Checkpoint, // Inactivity #[serde(with = "ssz_types::serde_utils::quoted_u64_var_list")] #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] - pub inactivity_scores: VariableList, + #[test_random(default)] + pub inactivity_scores: List, // Light-client sync committees #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] + #[metastruct(exclude_from(tree_lists))] pub current_sync_committee: Arc>, #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] + #[metastruct(exclude_from(tree_lists))] pub next_sync_committee: Arc>, // Execution @@ -305,89 +432,85 @@ where only(Merge), partial_getter(rename = "latest_execution_payload_header_merge") )] + #[metastruct(exclude_from(tree_lists))] pub latest_execution_payload_header: ExecutionPayloadHeaderMerge, #[superstruct( only(Capella), partial_getter(rename = "latest_execution_payload_header_capella") )] + #[metastruct(exclude_from(tree_lists))] pub latest_execution_payload_header: ExecutionPayloadHeaderCapella, #[superstruct( only(Deneb), partial_getter(rename = "latest_execution_payload_header_deneb") )] + #[metastruct(exclude_from(tree_lists))] pub latest_execution_payload_header: ExecutionPayloadHeaderDeneb, #[superstruct( only(Electra), partial_getter(rename = "latest_execution_payload_header_electra") )] + #[metastruct(exclude_from(tree_lists))] pub latest_execution_payload_header: ExecutionPayloadHeaderElectra, // Capella #[superstruct(only(Capella, Deneb, Electra), partial_getter(copy))] #[serde(with = "serde_utils::quoted_u64")] + #[metastruct(exclude_from(tree_lists))] pub next_withdrawal_index: u64, #[superstruct(only(Capella, Deneb, Electra), partial_getter(copy))] #[serde(with = "serde_utils::quoted_u64")] + #[metastruct(exclude_from(tree_lists))] pub next_withdrawal_validator_index: u64, // Deep history valid from Capella onwards. #[superstruct(only(Capella, Deneb, Electra))] - pub historical_summaries: VariableList, + #[test_random(default)] + pub historical_summaries: List, // Caching (not in the spec) #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] #[test_random(default)] - #[derivative(Clone(clone_with = "clone_default"))] + #[metastruct(exclude)] pub total_active_balance: Option<(Epoch, u64)>, #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] #[test_random(default)] - #[derivative(Clone(clone_with = "clone_default"))] - pub progressive_balances_cache: ProgressiveBalancesCache, + #[metastruct(exclude)] + pub committee_caches: [Arc; CACHED_EPOCHS], #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] #[test_random(default)] - #[derivative(Clone(clone_with = "clone_default"))] - pub committee_caches: [CommitteeCache; CACHED_EPOCHS], + #[metastruct(exclude)] + pub progressive_balances_cache: ProgressiveBalancesCache, #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] #[test_random(default)] - #[derivative(Clone(clone_with = "clone_default"))] + #[metastruct(exclude)] pub pubkey_cache: PubkeyCache, #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] #[test_random(default)] - #[derivative(Clone(clone_with = "clone_default"))] + #[metastruct(exclude)] pub exit_cache: ExitCache, #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] #[test_random(default)] - #[derivative(Clone(clone_with = "clone_default"))] + #[metastruct(exclude)] pub slashings_cache: SlashingsCache, /// Epoch cache of values that are useful for block processing that are static over an epoch. #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] #[tree_hash(skip_hashing)] #[test_random(default)] + #[metastruct(exclude)] pub epoch_cache: EpochCache, - #[serde(skip_serializing, skip_deserializing)] - #[ssz(skip_serializing, skip_deserializing)] - #[tree_hash(skip_hashing)] - #[test_random(default)] - #[derivative(Clone(clone_with = "clone_default"))] - pub tree_hash_cache: BeaconTreeHashCache, -} - -impl Clone for BeaconState { - fn clone(&self) -> Self { - self.clone_with(CloneConfig::all()) - } } impl BeaconState { @@ -395,6 +518,7 @@ impl BeaconState { /// /// Not a complete genesis state, see `initialize_beacon_state_from_eth1`. pub fn new(genesis_time: u64, eth1_data: Eth1Data, spec: &ChainSpec) -> Self { + let default_committee_cache = Arc::new(CommitteeCache::default()); BeaconState::Base(BeaconStateBase { // Versioning genesis_time, @@ -408,28 +532,28 @@ impl BeaconState { // History latest_block_header: BeaconBlock::::empty(spec).temporary_block_header(), - block_roots: FixedVector::from_elem(Hash256::zero()), - state_roots: FixedVector::from_elem(Hash256::zero()), - historical_roots: VariableList::empty(), + block_roots: Vector::default(), + state_roots: Vector::default(), + historical_roots: List::default(), // Eth1 eth1_data, - eth1_data_votes: VariableList::empty(), + eth1_data_votes: List::default(), eth1_deposit_index: 0, // Validator registry - validators: VariableList::empty(), // Set later. - balances: VariableList::empty(), // Set later. + validators: List::default(), // Set later. + balances: List::default(), // Set later. // Randomness - randao_mixes: FixedVector::from_elem(Hash256::zero()), + randao_mixes: Vector::default(), // Slashings - slashings: FixedVector::from_elem(0), + slashings: Vector::default(), // Attestations - previous_epoch_attestations: VariableList::empty(), - current_epoch_attestations: VariableList::empty(), + previous_epoch_attestations: List::default(), + current_epoch_attestations: List::default(), // Finality justification_bits: BitVector::new(), @@ -441,15 +565,14 @@ impl BeaconState { total_active_balance: None, progressive_balances_cache: <_>::default(), committee_caches: [ - CommitteeCache::default(), - CommitteeCache::default(), - CommitteeCache::default(), + default_committee_cache.clone(), + default_committee_cache.clone(), + default_committee_cache, ], pubkey_cache: PubkeyCache::default(), exit_cache: ExitCache::default(), slashings_cache: SlashingsCache::default(), epoch_cache: EpochCache::default(), - tree_hash_cache: <_>::default(), }) } @@ -485,30 +608,6 @@ impl BeaconState { } } - /// Specialised deserialisation method that uses the `ChainSpec` as context. - #[allow(clippy::arithmetic_side_effects)] - pub fn from_ssz_bytes(bytes: &[u8], spec: &ChainSpec) -> Result { - // Slot is after genesis_time (u64) and genesis_validators_root (Hash256). - let slot_start = ::ssz_fixed_len() + ::ssz_fixed_len(); - let slot_end = slot_start + ::ssz_fixed_len(); - - let slot_bytes = bytes - .get(slot_start..slot_end) - .ok_or(DecodeError::InvalidByteLength { - len: bytes.len(), - expected: slot_end, - })?; - - let slot = Slot::from_ssz_bytes(slot_bytes)?; - let fork_at_slot = spec.fork_name_at_slot::(slot); - - Ok(map_fork_name!( - fork_at_slot, - Self, - <_>::from_ssz_bytes(bytes)? - )) - } - /// Returns the `tree_hash_root` of the state. /// /// Spec v0.12.1 @@ -516,11 +615,15 @@ impl BeaconState { Hash256::from_slice(&self.tree_hash_root()[..]) } - pub fn historical_batch(&self) -> HistoricalBatch { - HistoricalBatch { + pub fn historical_batch(&mut self) -> Result, Error> { + // Updating before cloning makes the clone cheap and saves repeated hashing. + self.block_roots_mut().apply_updates()?; + self.state_roots_mut().apply_updates()?; + + Ok(HistoricalBatch { block_roots: self.block_roots().clone(), state_roots: self.state_roots().clone(), - } + }) } /// This method ensures the state's pubkey cache is fully up-to-date before checking if the validator @@ -1079,8 +1182,9 @@ impl BeaconState { } /// Fill `randao_mixes` with - pub fn fill_randao_mixes_with(&mut self, index_root: Hash256) { - *self.randao_mixes_mut() = FixedVector::from_elem(index_root); + pub fn fill_randao_mixes_with(&mut self, index_root: Hash256) -> Result<(), Error> { + *self.randao_mixes_mut() = Vector::from_elem(index_root)?; + Ok(()) } /// Safely obtains the index for `randao_mixes` @@ -1213,7 +1317,7 @@ impl BeaconState { } /// Get a reference to the entire `slashings` vector. - pub fn get_all_slashings(&self) -> &[u64] { + pub fn get_all_slashings(&self) -> &Vector { self.slashings() } @@ -1237,45 +1341,25 @@ impl BeaconState { } /// Convenience accessor for validators and balances simultaneously. - pub fn validators_and_balances_and_progressive_balances_mut( - &mut self, + pub fn validators_and_balances_and_progressive_balances_mut<'a>( + &'a mut self, ) -> ( - &mut Validators, - &mut Balances, - &mut ProgressiveBalancesCache, + &'a mut Validators, + &'a mut Balances, + &'a mut ProgressiveBalancesCache, ) { - match self { - BeaconState::Base(state) => ( - &mut state.validators, - &mut state.balances, - &mut state.progressive_balances_cache, - ), - BeaconState::Altair(state) => ( - &mut state.validators, - &mut state.balances, - &mut state.progressive_balances_cache, - ), - BeaconState::Merge(state) => ( - &mut state.validators, - &mut state.balances, - &mut state.progressive_balances_cache, - ), - BeaconState::Capella(state) => ( - &mut state.validators, - &mut state.balances, - &mut state.progressive_balances_cache, - ), - BeaconState::Deneb(state) => ( - &mut state.validators, - &mut state.balances, - &mut state.progressive_balances_cache, - ), - BeaconState::Electra(state) => ( - &mut state.validators, - &mut state.balances, - &mut state.progressive_balances_cache, - ), - } + map_beacon_state_ref_mut_into_beacon_state_ref!(&'a _, self.to_mut(), |inner, cons| { + if false { + cons(&*inner); + unreachable!() + } else { + ( + &mut inner.validators, + &mut inner.balances, + &mut inner.progressive_balances_cache, + ) + } + }) } #[allow(clippy::type_complexity)] @@ -1285,9 +1369,9 @@ impl BeaconState { ( &mut Validators, &mut Balances, - &VariableList, - &VariableList, - &mut VariableList, + &List, + &List, + &mut List, &mut ProgressiveBalancesCache, &mut ExitCache, &mut EpochCache, @@ -1349,6 +1433,13 @@ impl BeaconState { } } + /// Get a mutable reference to the balance of a single validator. + pub fn get_balance_mut(&mut self, validator_index: usize) -> Result<&mut u64, Error> { + self.balances_mut() + .get_mut(validator_index) + .ok_or(Error::BalancesOutOfBounds(validator_index)) + } + /// Generate a seed for the given `epoch`. pub fn get_seed( &self, @@ -1398,6 +1489,16 @@ impl BeaconState { .ok_or(Error::UnknownValidator(validator_index)) } + /// Safe copy-on-write accessor for the `validators` list. + pub fn get_validator_cow( + &mut self, + validator_index: usize, + ) -> Result, Error> { + self.validators_mut() + .get_cow(validator_index) + .ok_or(Error::UnknownValidator(validator_index)) + } + /// Return the effective balance for a validator with the given `validator_index`. pub fn get_effective_balance(&self, validator_index: usize) -> Result { self.get_validator(validator_index) @@ -1423,13 +1524,6 @@ impl BeaconState { .ok_or(Error::InactivityScoresOutOfBounds(validator_index)) } - /// Get a mutable reference to the balance of a single validator. - pub fn get_balance_mut(&mut self, validator_index: usize) -> Result<&mut u64, Error> { - self.balances_mut() - .get_mut(validator_index) - .ok_or(Error::BalancesOutOfBounds(validator_index)) - } - /// Return the epoch at which an activation or exit triggered in ``epoch`` takes effect. /// /// Spec v0.12.1 @@ -1575,7 +1669,7 @@ impl BeaconState { epoch: Epoch, previous_epoch: Epoch, current_epoch: Epoch, - ) -> Result<&mut VariableList, Error> { + ) -> Result<&mut List, Error> { if epoch == current_epoch { match self { BeaconState::Base(_) => Err(BeaconStateError::IncorrectStateVariant), @@ -1659,7 +1753,6 @@ impl BeaconState { self.drop_committee_cache(RelativeEpoch::Current)?; self.drop_committee_cache(RelativeEpoch::Next)?; self.drop_pubkey_cache(); - self.drop_tree_hash_cache(); self.drop_progressive_balances_cache(); *self.exit_cache_mut() = ExitCache::default(); *self.slashings_cache_mut() = SlashingsCache::default(); @@ -1718,7 +1811,7 @@ impl BeaconState { &self, epoch: Epoch, spec: &ChainSpec, - ) -> Result { + ) -> Result, Error> { CommitteeCache::initialized(self, epoch, spec) } @@ -1732,7 +1825,7 @@ impl BeaconState { self.committee_caches_mut().rotate_left(1); let next = Self::committee_cache_index(RelativeEpoch::Next); - *self.committee_cache_at_index_mut(next)? = CommitteeCache::default(); + *self.committee_cache_at_index_mut(next)? = Arc::new(CommitteeCache::default()); Ok(()) } @@ -1747,21 +1840,24 @@ impl BeaconState { /// Get the committee cache for some `slot`. /// /// Return an error if the cache for the slot's epoch is not initialized. - fn committee_cache_at_slot(&self, slot: Slot) -> Result<&CommitteeCache, Error> { + fn committee_cache_at_slot(&self, slot: Slot) -> Result<&Arc, Error> { let epoch = slot.epoch(E::slots_per_epoch()); let relative_epoch = RelativeEpoch::from_epoch(self.current_epoch(), epoch)?; self.committee_cache(relative_epoch) } /// Get the committee cache at a given index. - fn committee_cache_at_index(&self, index: usize) -> Result<&CommitteeCache, Error> { + fn committee_cache_at_index(&self, index: usize) -> Result<&Arc, Error> { self.committee_caches() .get(index) .ok_or(Error::CommitteeCachesOutOfBounds(index)) } /// Get a mutable reference to the committee cache at a given index. - fn committee_cache_at_index_mut(&mut self, index: usize) -> Result<&mut CommitteeCache, Error> { + fn committee_cache_at_index_mut( + &mut self, + index: usize, + ) -> Result<&mut Arc, Error> { self.committee_caches_mut() .get_mut(index) .ok_or(Error::CommitteeCachesOutOfBounds(index)) @@ -1769,7 +1865,10 @@ impl BeaconState { /// Returns the cache for some `RelativeEpoch`. Returns an error if the cache has not been /// initialized. - pub fn committee_cache(&self, relative_epoch: RelativeEpoch) -> Result<&CommitteeCache, Error> { + pub fn committee_cache( + &self, + relative_epoch: RelativeEpoch, + ) -> Result<&Arc, Error> { let i = Self::committee_cache_index(relative_epoch); let cache = self.committee_cache_at_index(i)?; @@ -1780,30 +1879,10 @@ impl BeaconState { } } - /// Returns the cache for some `RelativeEpoch`, replacing the existing cache with an - /// un-initialized cache. Returns an error if the existing cache has not been initialized. - pub fn take_committee_cache( - &mut self, - relative_epoch: RelativeEpoch, - ) -> Result { - let i = Self::committee_cache_index(relative_epoch); - let current_epoch = self.current_epoch(); - let cache = self - .committee_caches_mut() - .get_mut(i) - .ok_or(Error::CommitteeCachesOutOfBounds(i))?; - - if cache.is_initialized_at(relative_epoch.into_epoch(current_epoch)) { - Ok(mem::take(cache)) - } else { - Err(Error::CommitteeCacheUninitialized(Some(relative_epoch))) - } - } - /// Drops the cache, leaving it in an uninitialized state. pub fn drop_committee_cache(&mut self, relative_epoch: RelativeEpoch) -> Result<(), Error> { *self.committee_cache_at_index_mut(Self::committee_cache_index(relative_epoch))? = - CommitteeCache::default(); + Arc::new(CommitteeCache::default()); Ok(()) } @@ -1813,13 +1892,11 @@ impl BeaconState { /// never re-add a pubkey. pub fn update_pubkey_cache(&mut self) -> Result<(), Error> { let mut pubkey_cache = mem::take(self.pubkey_cache_mut()); - for (i, validator) in self - .validators() - .iter() - .enumerate() - .skip(pubkey_cache.len()) - { - let success = pubkey_cache.insert(validator.pubkey, i); + let start_index = pubkey_cache.len(); + + for (i, validator) in self.validators().iter_from(start_index)?.enumerate() { + let index = start_index.safe_add(i)?; + let success = pubkey_cache.insert(validator.pubkey, index); if !success { return Err(Error::PubkeyCacheInconsistent); } @@ -1834,96 +1911,71 @@ impl BeaconState { *self.pubkey_cache_mut() = PubkeyCache::default() } + pub fn has_pending_mutations(&self) -> bool { + let mut any_pending_mutations = false; + match &self { + Self::Base(self_inner) => { + map_beacon_state_base_tree_list_fields_immutable!(self_inner, |_, self_field| { + any_pending_mutations |= self_field.has_pending_updates(); + }); + } + Self::Altair(self_inner) => { + map_beacon_state_altair_tree_list_fields_immutable!(self_inner, |_, self_field| { + any_pending_mutations |= self_field.has_pending_updates(); + }); + } + Self::Merge(self_inner) => { + map_beacon_state_bellatrix_tree_list_fields_immutable!( + self_inner, + |_, self_field| { + any_pending_mutations |= self_field.has_pending_updates(); + } + ); + } + Self::Capella(self_inner) => { + map_beacon_state_capella_tree_list_fields_immutable!( + self_inner, + |_, self_field| { + any_pending_mutations |= self_field.has_pending_updates(); + } + ); + } + Self::Deneb(self_inner) => { + map_beacon_state_deneb_tree_list_fields_immutable!(self_inner, |_, self_field| { + any_pending_mutations |= self_field.has_pending_updates(); + }); + } + Self::Electra(self_inner) => { + map_beacon_state_electra_tree_list_fields_immutable!( + self_inner, + |_, self_field| { + any_pending_mutations |= self_field.has_pending_updates(); + } + ); + } + }; + any_pending_mutations + } + /// Completely drops the `progressive_balances_cache` cache, replacing it with a new, empty cache. fn drop_progressive_balances_cache(&mut self) { *self.progressive_balances_cache_mut() = ProgressiveBalancesCache::default(); } - /// Initialize but don't fill the tree hash cache, if it isn't already initialized. - pub fn initialize_tree_hash_cache(&mut self) { - if !self.tree_hash_cache().is_initialized() { - *self.tree_hash_cache_mut() = BeaconTreeHashCache::new(self) - } - } - /// Compute the tree hash root of the state using the tree hash cache. /// /// Initialize the tree hash cache if it isn't already initialized. pub fn update_tree_hash_cache(&mut self) -> Result { - self.initialize_tree_hash_cache(); - - let cache = self.tree_hash_cache_mut().take(); - - if let Some(mut cache) = cache { - // Note: we return early if the tree hash fails, leaving `self.tree_hash_cache` as - // None. There's no need to keep a cache that fails. - let root = cache.recalculate_tree_hash_root(self)?; - self.tree_hash_cache_mut().restore(cache); - Ok(root) - } else { - Err(Error::TreeHashCacheNotInitialized) - } + self.apply_pending_mutations()?; + Ok(self.tree_hash_root()) } /// Compute the tree hash root of the validators using the tree hash cache. /// /// Initialize the tree hash cache if it isn't already initialized. pub fn update_validators_tree_hash_cache(&mut self) -> Result { - self.initialize_tree_hash_cache(); - - let cache = self.tree_hash_cache_mut().take(); - - if let Some(mut cache) = cache { - // Note: we return early if the tree hash fails, leaving `self.tree_hash_cache` as - // None. There's no need to keep a cache that fails. - let root = cache.recalculate_validators_tree_hash_root(self.validators())?; - self.tree_hash_cache_mut().restore(cache); - Ok(root) - } else { - Err(Error::TreeHashCacheNotInitialized) - } - } - - /// Completely drops the tree hash cache, replacing it with a new, empty cache. - pub fn drop_tree_hash_cache(&mut self) { - self.tree_hash_cache_mut().uninitialize(); - } - - /// Clone the state whilst preserving only the selected caches. - pub fn clone_with(&self, config: CloneConfig) -> Self { - let mut res = match self { - BeaconState::Base(inner) => BeaconState::Base(inner.clone()), - BeaconState::Altair(inner) => BeaconState::Altair(inner.clone()), - BeaconState::Merge(inner) => BeaconState::Merge(inner.clone()), - BeaconState::Capella(inner) => BeaconState::Capella(inner.clone()), - BeaconState::Deneb(inner) => BeaconState::Deneb(inner.clone()), - BeaconState::Electra(inner) => BeaconState::Electra(inner.clone()), - }; - if config.committee_caches { - res.committee_caches_mut() - .clone_from(self.committee_caches()); - *res.total_active_balance_mut() = *self.total_active_balance(); - } - if config.pubkey_cache { - *res.pubkey_cache_mut() = self.pubkey_cache().clone(); - } - if config.exit_cache { - *res.exit_cache_mut() = self.exit_cache().clone(); - } - if config.slashings_cache { - *res.slashings_cache_mut() = self.slashings_cache().clone(); - } - if config.tree_hash_cache { - *res.tree_hash_cache_mut() = self.tree_hash_cache().clone(); - } - if config.progressive_balances_cache { - *res.progressive_balances_cache_mut() = self.progressive_balances_cache().clone(); - } - res - } - - pub fn clone_with_only_committee_caches(&self) -> Self { - self.clone_with(CloneConfig::committee_caches_only()) + self.validators_mut().apply_updates()?; + Ok(self.validators().tree_hash_root()) } /// Passing `previous_epoch` to this function rather than computing it internally provides @@ -1979,10 +2031,177 @@ impl BeaconState { self.epoch_cache().get_base_reward(validator_index) } - pub fn compute_merkle_proof( - &mut self, - generalized_index: usize, - ) -> Result, Error> { + #[allow(clippy::arithmetic_side_effects)] + pub fn rebase_on(&mut self, base: &Self, spec: &ChainSpec) -> Result<(), Error> { + // Required for macros (which use type-hints internally). + + match (&mut *self, base) { + (Self::Base(self_inner), Self::Base(base_inner)) => { + bimap_beacon_state_base_tree_list_fields!( + self_inner, + base_inner, + |_, self_field, base_field| { self_field.rebase_on(base_field) } + ); + } + (Self::Base(_), _) => (), + (Self::Altair(self_inner), Self::Altair(base_inner)) => { + bimap_beacon_state_altair_tree_list_fields!( + self_inner, + base_inner, + |_, self_field, base_field| { self_field.rebase_on(base_field) } + ); + } + (Self::Altair(_), _) => (), + (Self::Merge(self_inner), Self::Merge(base_inner)) => { + bimap_beacon_state_bellatrix_tree_list_fields!( + self_inner, + base_inner, + |_, self_field, base_field| { self_field.rebase_on(base_field) } + ); + } + (Self::Merge(_), _) => (), + (Self::Capella(self_inner), Self::Capella(base_inner)) => { + bimap_beacon_state_capella_tree_list_fields!( + self_inner, + base_inner, + |_, self_field, base_field| { self_field.rebase_on(base_field) } + ); + } + (Self::Capella(_), _) => (), + (Self::Deneb(self_inner), Self::Deneb(base_inner)) => { + bimap_beacon_state_deneb_tree_list_fields!( + self_inner, + base_inner, + |_, self_field, base_field| { self_field.rebase_on(base_field) } + ); + } + (Self::Deneb(_), _) => (), + (Self::Electra(self_inner), Self::Electra(base_inner)) => { + bimap_beacon_state_electra_tree_list_fields!( + self_inner, + base_inner, + |_, self_field, base_field| { self_field.rebase_on(base_field) } + ); + } + (Self::Electra(_), _) => (), + } + + // Use sync committees from `base` if they are equal. + if let Ok(current_sync_committee) = self.current_sync_committee_mut() { + if let Ok(base_sync_committee) = base.current_sync_committee() { + if current_sync_committee == base_sync_committee { + *current_sync_committee = base_sync_committee.clone(); + } + } + } + if let Ok(next_sync_committee) = self.next_sync_committee_mut() { + if let Ok(base_sync_committee) = base.next_sync_committee() { + if next_sync_committee == base_sync_committee { + *next_sync_committee = base_sync_committee.clone(); + } + } + } + + // Rebase caches like the committee caches and the pubkey cache, which are expensive to + // rebuild and likely to be re-usable from the base state. + self.rebase_caches_on(base, spec)?; + + Ok(()) + } + + pub fn rebase_caches_on(&mut self, base: &Self, spec: &ChainSpec) -> Result<(), Error> { + // Use pubkey cache from `base` if it contains superior information (likely if our cache is + // uninitialized). + let num_validators = self.validators().len(); + let pubkey_cache = self.pubkey_cache_mut(); + let base_pubkey_cache = base.pubkey_cache(); + if pubkey_cache.len() < base_pubkey_cache.len() && pubkey_cache.len() < num_validators { + *pubkey_cache = base_pubkey_cache.clone(); + } + + // Use committee caches from `base` if they are relevant. + let epochs = [ + self.previous_epoch(), + self.current_epoch(), + self.next_epoch()?, + ]; + for (index, epoch) in epochs.into_iter().enumerate() { + if let Ok(base_relative_epoch) = RelativeEpoch::from_epoch(base.current_epoch(), epoch) + { + *self.committee_cache_at_index_mut(index)? = + base.committee_cache(base_relative_epoch)?.clone(); + + // Ensure total active balance cache remains built whenever current committee + // cache is built. + if epoch == self.current_epoch() { + self.build_total_active_balance_cache(spec)?; + } + } + } + + Ok(()) + } +} + +impl BeaconState { + /// The number of fields of the `BeaconState` rounded up to the nearest power of two. + /// + /// This is relevant to tree-hashing of the `BeaconState`. + /// + /// We assume this value is stable across forks. This assumption is checked in the + /// `check_num_fields_pow2` test. + pub const NUM_FIELDS_POW2: usize = BeaconStateMerge::::NUM_FIELDS.next_power_of_two(); + + /// Specialised deserialisation method that uses the `ChainSpec` as context. + #[allow(clippy::arithmetic_side_effects)] + pub fn from_ssz_bytes(bytes: &[u8], spec: &ChainSpec) -> Result { + // Slot is after genesis_time (u64) and genesis_validators_root (Hash256). + let slot_start = ::ssz_fixed_len() + ::ssz_fixed_len(); + let slot_end = slot_start + ::ssz_fixed_len(); + + let slot_bytes = bytes + .get(slot_start..slot_end) + .ok_or(DecodeError::InvalidByteLength { + len: bytes.len(), + expected: slot_end, + })?; + + let slot = Slot::from_ssz_bytes(slot_bytes)?; + let fork_at_slot = spec.fork_name_at_slot::(slot); + + Ok(map_fork_name!( + fork_at_slot, + Self, + <_>::from_ssz_bytes(bytes)? + )) + } + + #[allow(clippy::arithmetic_side_effects)] + pub fn apply_pending_mutations(&mut self) -> Result<(), Error> { + match self { + Self::Base(inner) => { + map_beacon_state_base_tree_list_fields!(inner, |_, x| { x.apply_updates() }) + } + Self::Altair(inner) => { + map_beacon_state_altair_tree_list_fields!(inner, |_, x| { x.apply_updates() }) + } + Self::Merge(inner) => { + map_beacon_state_bellatrix_tree_list_fields!(inner, |_, x| { x.apply_updates() }) + } + Self::Capella(inner) => { + map_beacon_state_capella_tree_list_fields!(inner, |_, x| { x.apply_updates() }) + } + Self::Deneb(inner) => { + map_beacon_state_deneb_tree_list_fields!(inner, |_, x| { x.apply_updates() }) + } + Self::Electra(inner) => { + map_beacon_state_electra_tree_list_fields!(inner, |_, x| { x.apply_updates() }) + } + } + Ok(()) + } + + pub fn compute_merkle_proof(&self, generalized_index: usize) -> Result, Error> { // 1. Convert generalized index to field index. let field_index = match generalized_index { light_client_update::CURRENT_SYNC_COMMITTEE_INDEX @@ -1992,7 +2211,7 @@ impl BeaconState { // in the `BeaconState`: // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#beaconstate generalized_index - .checked_sub(tree_hash_cache::NUM_BEACON_STATE_HASH_TREE_ROOT_LEAVES) + .checked_sub(Self::NUM_FIELDS_POW2) .ok_or(Error::IndexNotSupported(generalized_index))? } light_client_update::FINALIZED_ROOT_INDEX => { @@ -2002,20 +2221,47 @@ impl BeaconState { // Subtract off the internal nodes. Result should be 105/2 - 32 = 20 which matches // position of `finalized_checkpoint` in `BeaconState`. finalized_checkpoint_generalized_index - .checked_sub(tree_hash_cache::NUM_BEACON_STATE_HASH_TREE_ROOT_LEAVES) + .checked_sub(Self::NUM_FIELDS_POW2) .ok_or(Error::IndexNotSupported(generalized_index))? } _ => return Err(Error::IndexNotSupported(generalized_index)), }; // 2. Get all `BeaconState` leaves. - self.initialize_tree_hash_cache(); - let mut cache = self - .tree_hash_cache_mut() - .take() - .ok_or(Error::TreeHashCacheNotInitialized)?; - let leaves = cache.recalculate_tree_hash_leaves(self)?; - self.tree_hash_cache_mut().restore(cache); + let mut leaves = vec![]; + #[allow(clippy::arithmetic_side_effects)] + match self { + BeaconState::Base(state) => { + map_beacon_state_base_fields!(state, |_, field| { + leaves.push(field.tree_hash_root()); + }); + } + BeaconState::Altair(state) => { + map_beacon_state_altair_fields!(state, |_, field| { + leaves.push(field.tree_hash_root()); + }); + } + BeaconState::Merge(state) => { + map_beacon_state_bellatrix_fields!(state, |_, field| { + leaves.push(field.tree_hash_root()); + }); + } + BeaconState::Capella(state) => { + map_beacon_state_capella_fields!(state, |_, field| { + leaves.push(field.tree_hash_root()); + }); + } + BeaconState::Deneb(state) => { + map_beacon_state_deneb_fields!(state, |_, field| { + leaves.push(field.tree_hash_root()); + }); + } + BeaconState::Electra(state) => { + map_beacon_state_electra_fields!(state, |_, field| { + leaves.push(field.tree_hash_root()); + }); + } + }; // 3. Make deposit tree. // Use the depth of the `BeaconState` fields (i.e. `log2(32) = 5`). @@ -2074,9 +2320,10 @@ impl From for Error { } } -/// Helper function for "cloning" a field by using its default value. -fn clone_default(_value: &T) -> T { - T::default() +impl From for Error { + fn from(e: milhouse::Error) -> Self { + Self::MilhouseError(e) + } } impl CompareFields for BeaconState { diff --git a/consensus/types/src/beacon_state/clone_config.rs b/consensus/types/src/beacon_state/clone_config.rs deleted file mode 100644 index 27e066d5db6..00000000000 --- a/consensus/types/src/beacon_state/clone_config.rs +++ /dev/null @@ -1,47 +0,0 @@ -/// Configuration struct for controlling which caches of a `BeaconState` should be cloned. -#[derive(Debug, Default, PartialEq, Eq, Clone, Copy)] -pub struct CloneConfig { - pub committee_caches: bool, - pub pubkey_cache: bool, - pub exit_cache: bool, - pub slashings_cache: bool, - pub tree_hash_cache: bool, - pub progressive_balances_cache: bool, -} - -impl CloneConfig { - pub fn all() -> Self { - Self { - committee_caches: true, - pubkey_cache: true, - exit_cache: true, - slashings_cache: true, - tree_hash_cache: true, - progressive_balances_cache: true, - } - } - - pub fn none() -> Self { - Self::default() - } - - pub fn committee_caches_only() -> Self { - Self { - committee_caches: true, - ..Self::none() - } - } -} - -#[cfg(test)] -mod test { - use super::*; - - #[test] - fn sanity() { - assert!(CloneConfig::all().pubkey_cache); - assert!(!CloneConfig::none().tree_hash_cache); - assert!(CloneConfig::committee_caches_only().committee_caches); - assert!(!CloneConfig::committee_caches_only().exit_cache); - } -} diff --git a/consensus/types/src/beacon_state/committee_cache.rs b/consensus/types/src/beacon_state/committee_cache.rs index a6b12cf5af3..7913df8e00e 100644 --- a/consensus/types/src/beacon_state/committee_cache.rs +++ b/consensus/types/src/beacon_state/committee_cache.rs @@ -8,6 +8,7 @@ use serde::{Deserialize, Serialize}; use ssz::{four_byte_option_impl, Decode, DecodeError, Encode}; use ssz_derive::{Decode, Encode}; use std::ops::Range; +use std::sync::Arc; use swap_or_not_shuffle::shuffle_list; mod tests; @@ -65,7 +66,7 @@ impl CommitteeCache { state: &BeaconState, epoch: Epoch, spec: &ChainSpec, - ) -> Result { + ) -> Result, Error> { // Check that the cache is being built for an in-range epoch. // // We allow caches to be constructed for historic epochs, per: @@ -115,13 +116,13 @@ impl CommitteeCache { .ok_or(Error::ShuffleIndexOutOfBounds(v))? = NonZeroUsize::new(i + 1).into(); } - Ok(CommitteeCache { + Ok(Arc::new(CommitteeCache { initialized_epoch: Some(epoch), shuffling, shuffling_positions, committees_per_slot, slots_per_epoch: E::slots_per_epoch(), - }) + })) } /// Returns `true` if the cache has been initialized at the supplied `epoch`. diff --git a/consensus/types/src/beacon_state/committee_cache/tests.rs b/consensus/types/src/beacon_state/committee_cache/tests.rs index a5effb9363b..a2274765691 100644 --- a/consensus/types/src/beacon_state/committee_cache/tests.rs +++ b/consensus/types/src/beacon_state/committee_cache/tests.rs @@ -92,7 +92,7 @@ async fn shuffles_for_the_right_epoch() { .map(|i| Hash256::from_low_u64_be(i as u64)) .collect(); - *state.randao_mixes_mut() = FixedVector::from(distinct_hashes); + *state.randao_mixes_mut() = Vector::try_from_iter(distinct_hashes).unwrap(); let previous_seed = state .get_seed(state.previous_epoch(), Domain::BeaconAttester, spec) diff --git a/consensus/types/src/beacon_state/exit_cache.rs b/consensus/types/src/beacon_state/exit_cache.rs index bda788e63b9..0bb984b6676 100644 --- a/consensus/types/src/beacon_state/exit_cache.rs +++ b/consensus/types/src/beacon_state/exit_cache.rs @@ -1,10 +1,9 @@ use super::{BeaconStateError, ChainSpec, Epoch, Validator}; use safe_arith::SafeArith; -use serde::{Deserialize, Serialize}; use std::cmp::Ordering; /// Map from exit epoch to the number of validators with that exit epoch. -#[derive(Debug, Default, Clone, PartialEq, Serialize, Deserialize)] +#[derive(Debug, Default, Clone, PartialEq)] pub struct ExitCache { /// True if the cache has been initialized. initialized: bool, @@ -16,7 +15,11 @@ pub struct ExitCache { impl ExitCache { /// Initialize a new cache for the given list of validators. - pub fn new(validators: &[Validator], spec: &ChainSpec) -> Result { + pub fn new<'a, V, I>(validators: V, spec: &ChainSpec) -> Result + where + V: IntoIterator, + I: ExactSizeIterator + Iterator, + { let mut exit_cache = ExitCache { initialized: true, max_exit_epoch: Epoch::new(0), @@ -24,7 +27,7 @@ impl ExitCache { }; // Add all validators with a non-default exit epoch to the cache. validators - .iter() + .into_iter() .filter(|validator| validator.exit_epoch != spec.far_future_epoch) .try_for_each(|validator| exit_cache.record_validator_exit(validator.exit_epoch))?; Ok(exit_cache) diff --git a/consensus/types/src/beacon_state/iter.rs b/consensus/types/src/beacon_state/iter.rs index 2d3ad02c836..2caa0365e01 100644 --- a/consensus/types/src/beacon_state/iter.rs +++ b/consensus/types/src/beacon_state/iter.rs @@ -74,7 +74,7 @@ mod test { let mut state: BeaconState = BeaconState::new(0, <_>::default(), &spec); for i in 0..state.block_roots().len() { - state.block_roots_mut()[i] = root_slot(i).1; + *state.block_roots_mut().get_mut(i).unwrap() = root_slot(i).1; } assert_eq!( @@ -122,7 +122,7 @@ mod test { let mut state: BeaconState = BeaconState::new(0, <_>::default(), &spec); for i in 0..state.block_roots().len() { - state.block_roots_mut()[i] = root_slot(i).1; + *state.block_roots_mut().get_mut(i).unwrap() = root_slot(i).1; } assert_eq!( diff --git a/consensus/types/src/beacon_state/pubkey_cache.rs b/consensus/types/src/beacon_state/pubkey_cache.rs index 0b61ea3c5f8..d58dd7bc1dd 100644 --- a/consensus/types/src/beacon_state/pubkey_cache.rs +++ b/consensus/types/src/beacon_state/pubkey_cache.rs @@ -1,21 +1,21 @@ use crate::*; -use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use rpds::HashTrieMapSync as HashTrieMap; type ValidatorIndex = usize; #[allow(clippy::len_without_is_empty)] -#[derive(Debug, PartialEq, Clone, Default, Serialize, Deserialize)] +#[derive(Debug, PartialEq, Clone, Default)] pub struct PubkeyCache { - /// Maintain the number of keys added to the map. It is not sufficient to just use the HashMap - /// len, as it does not increase when duplicate keys are added. Duplicate keys are used during - /// testing. + /// Maintain the number of keys added to the map. It is not sufficient to just use the + /// HashTrieMap len, as it does not increase when duplicate keys are added. Duplicate keys are + /// used during testing. len: usize, - map: HashMap, + map: HashTrieMap, } impl PubkeyCache { /// Returns the number of validator indices added to the map so far. + #[allow(clippy::len_without_is_empty)] pub fn len(&self) -> ValidatorIndex { self.len } @@ -26,7 +26,7 @@ impl PubkeyCache { /// that an index is never skipped. pub fn insert(&mut self, pubkey: PublicKeyBytes, index: ValidatorIndex) -> bool { if index == self.len { - self.map.insert(pubkey, index); + self.map.insert_mut(pubkey, index); self.len = self .len .checked_add(1) diff --git a/consensus/types/src/beacon_state/slashings_cache.rs b/consensus/types/src/beacon_state/slashings_cache.rs index cfdc349f86c..45d8f7e2129 100644 --- a/consensus/types/src/beacon_state/slashings_cache.rs +++ b/consensus/types/src/beacon_state/slashings_cache.rs @@ -1,13 +1,13 @@ use crate::{BeaconStateError, Slot, Validator}; use arbitrary::Arbitrary; -use std::collections::HashSet; +use rpds::HashTrieSetSync as HashTrieSet; /// Persistent (cheap to clone) cache of all slashed validator indices. #[derive(Debug, Default, Clone, PartialEq, Arbitrary)] pub struct SlashingsCache { latest_block_slot: Option, #[arbitrary(default)] - slashed_validators: HashSet, + slashed_validators: HashTrieSet, } impl SlashingsCache { @@ -49,7 +49,7 @@ impl SlashingsCache { validator_index: usize, ) -> Result<(), BeaconStateError> { self.check_initialized(block_slot)?; - self.slashed_validators.insert(validator_index); + self.slashed_validators.insert_mut(validator_index); Ok(()) } diff --git a/consensus/types/src/beacon_state/tests.rs b/consensus/types/src/beacon_state/tests.rs index 00625a1788e..012c063afef 100644 --- a/consensus/types/src/beacon_state/tests.rs +++ b/consensus/types/src/beacon_state/tests.rs @@ -1,20 +1,14 @@ #![cfg(test)] -use crate::test_utils::*; -use beacon_chain::test_utils::{ - interop_genesis_state_with_eth1, test_spec, BeaconChainHarness, EphemeralHarnessType, - DEFAULT_ETH1_BLOCK_HASH, -}; +use crate::{test_utils::*, ForkName}; +use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType}; use beacon_chain::types::{ - test_utils::TestRandom, BeaconState, BeaconStateAltair, BeaconStateBase, BeaconStateError, - ChainSpec, CloneConfig, Domain, Epoch, EthSpec, FixedVector, Hash256, Keypair, MainnetEthSpec, - MinimalEthSpec, RelativeEpoch, Slot, + test_utils::TestRandom, BeaconState, BeaconStateAltair, BeaconStateBase, BeaconStateCapella, + BeaconStateDeneb, BeaconStateElectra, BeaconStateError, BeaconStateMerge, ChainSpec, Domain, + Epoch, EthSpec, Hash256, Keypair, MainnetEthSpec, MinimalEthSpec, RelativeEpoch, Slot, Vector, }; -use safe_arith::SafeArith; use ssz::Encode; -use state_processing::per_slot_processing; use std::ops::Mul; use swap_or_not_shuffle::compute_shuffled_index; -use tree_hash::TreeHash; pub const MAX_VALIDATOR_COUNT: usize = 129; pub const SLOT_OFFSET: Slot = Slot::new(1); @@ -101,7 +95,11 @@ async fn test_beacon_proposer_index() { // Test with two validators per slot, first validator has zero balance. let mut state = build_state::((E::slots_per_epoch() as usize).mul(2)).await; let slot0_candidate0 = ith_candidate(&state, Slot::new(0), 0, &spec); - state.validators_mut()[slot0_candidate0].effective_balance = 0; + state + .validators_mut() + .get_mut(slot0_candidate0) + .unwrap() + .effective_balance = 0; test(&state, Slot::new(0), 1); for i in 1..E::slots_per_epoch() { test(&state, Slot::from(i), 0); @@ -159,85 +157,6 @@ async fn cache_initialization() { test_cache_initialization(&mut state, RelativeEpoch::Next, &spec); } -fn test_clone_config(base_state: &BeaconState, clone_config: CloneConfig) { - let state = base_state.clone_with(clone_config); - if clone_config.committee_caches { - state - .committee_cache(RelativeEpoch::Previous) - .expect("committee cache exists"); - state - .committee_cache(RelativeEpoch::Current) - .expect("committee cache exists"); - state - .committee_cache(RelativeEpoch::Next) - .expect("committee cache exists"); - state - .total_active_balance() - .expect("total active balance exists"); - } else { - state - .committee_cache(RelativeEpoch::Previous) - .expect_err("shouldn't exist"); - state - .committee_cache(RelativeEpoch::Current) - .expect_err("shouldn't exist"); - state - .committee_cache(RelativeEpoch::Next) - .expect_err("shouldn't exist"); - } - if clone_config.pubkey_cache { - assert_ne!(state.pubkey_cache().len(), 0); - } else { - assert_eq!(state.pubkey_cache().len(), 0); - } - if clone_config.exit_cache { - state - .exit_cache() - .check_initialized() - .expect("exit cache exists"); - } else { - state - .exit_cache() - .check_initialized() - .expect_err("exit cache doesn't exist"); - } - if clone_config.tree_hash_cache { - assert!(state.tree_hash_cache().is_initialized()); - } else { - assert!( - !state.tree_hash_cache().is_initialized(), - "{:?}", - clone_config - ); - } -} - -#[tokio::test] -async fn clone_config() { - let spec = MinimalEthSpec::default_spec(); - - let mut state = build_state::(16).await; - - state.build_caches(&spec).unwrap(); - state - .update_tree_hash_cache() - .expect("should update tree hash cache"); - - let num_caches = 6; - let all_configs = (0..2u8.pow(num_caches)).map(|i| CloneConfig { - committee_caches: (i & 1) != 0, - pubkey_cache: ((i >> 1) & 1) != 0, - exit_cache: ((i >> 2) & 1) != 0, - slashings_cache: ((i >> 3) & 1) != 0, - tree_hash_cache: ((i >> 4) & 1) != 0, - progressive_balances_cache: ((i >> 5) & 1) != 0, - }); - - for config in all_configs { - test_clone_config(&state, config); - } -} - /// Tests committee-specific components #[cfg(test)] mod committees { @@ -328,10 +247,9 @@ mod committees { let harness = get_harness::(validator_count, slot).await; let mut new_head_state = harness.get_current_state(); - let distinct_hashes: Vec = (0..E::epochs_per_historical_vector()) - .map(|i| Hash256::from_low_u64_be(i as u64)) - .collect(); - *new_head_state.randao_mixes_mut() = FixedVector::from(distinct_hashes); + let distinct_hashes = + (0..E::epochs_per_historical_vector()).map(|i| Hash256::from_low_u64_be(i as u64)); + *new_head_state.randao_mixes_mut() = Vector::try_from_iter(distinct_hashes).unwrap(); new_head_state .force_build_committee_cache(RelativeEpoch::Previous, spec) @@ -487,120 +405,22 @@ fn decode_base_and_altair() { } #[test] -fn tree_hash_cache_linear_history() { - let mut rng = XorShiftRng::from_seed([42; 16]); - - let mut state: BeaconState = - BeaconState::Base(BeaconStateBase::random_for_test(&mut rng)); - - let root = state.update_tree_hash_cache().unwrap(); - - assert_eq!(root.as_bytes(), &state.tree_hash_root()[..]); - - /* - * A cache should hash twice without updating the slot. - */ - - assert_eq!( - state.update_tree_hash_cache().unwrap(), - root, - "tree hash result should be identical on the same slot" - ); - - /* - * A cache should not hash after updating the slot but not updating the state roots. - */ - - // The tree hash cache needs to be rebuilt since it was dropped when it failed. - state - .update_tree_hash_cache() - .expect("should rebuild cache"); - - *state.slot_mut() += 1; - - assert_eq!( - state.update_tree_hash_cache(), - Err(BeaconStateError::NonLinearTreeHashCacheHistory), - "should not build hash without updating the state root" - ); - - /* - * The cache should update if the slot and state root are updated. - */ - - // The tree hash cache needs to be rebuilt since it was dropped when it failed. - let root = state - .update_tree_hash_cache() - .expect("should rebuild cache"); - - *state.slot_mut() += 1; - state - .set_state_root(state.slot() - 1, root) - .expect("should set state root"); - - let root = state.update_tree_hash_cache().unwrap(); - assert_eq!(root.as_bytes(), &state.tree_hash_root()[..]); -} - -// Check how the cache behaves when there's a distance larger than `SLOTS_PER_HISTORICAL_ROOT` -// since its last update. -#[test] -fn tree_hash_cache_linear_history_long_skip() { - let validator_count = 128; - let keypairs = generate_deterministic_keypairs(validator_count); - - let spec = &test_spec::(); - - // This state has a cache that advances normally each slot. - let mut state: BeaconState = interop_genesis_state_with_eth1( - &keypairs, - 0, - Hash256::from_slice(DEFAULT_ETH1_BLOCK_HASH), - None, - spec, - ) - .unwrap(); - - state.update_tree_hash_cache().unwrap(); - - // This state retains its original cache until it is updated after a long skip. - let mut original_cache_state = state.clone(); - assert!(original_cache_state.tree_hash_cache().is_initialized()); - - // Advance the states to a slot beyond the historical state root limit, using the state root - // from the first state to avoid touching the original state's cache. - let start_slot = state.slot(); - let target_slot = start_slot - .safe_add(MinimalEthSpec::slots_per_historical_root() as u64 + 1) - .unwrap(); - - let mut prev_state_root; - while state.slot() < target_slot { - prev_state_root = state.update_tree_hash_cache().unwrap(); - per_slot_processing(&mut state, None, spec).unwrap(); - per_slot_processing(&mut original_cache_state, Some(prev_state_root), spec).unwrap(); +fn check_num_fields_pow2() { + use metastruct::NumFields; + pub type E = MainnetEthSpec; + + for fork_name in ForkName::list_all() { + let num_fields = match fork_name { + ForkName::Base => BeaconStateBase::::NUM_FIELDS, + ForkName::Altair => BeaconStateAltair::::NUM_FIELDS, + ForkName::Merge => BeaconStateMerge::::NUM_FIELDS, + ForkName::Capella => BeaconStateCapella::::NUM_FIELDS, + ForkName::Deneb => BeaconStateDeneb::::NUM_FIELDS, + ForkName::Electra => BeaconStateElectra::::NUM_FIELDS, + }; + assert_eq!( + num_fields.next_power_of_two(), + BeaconState::::NUM_FIELDS_POW2 + ); } - - // The state with the original cache should still be initialized at the starting slot. - assert_eq!( - original_cache_state - .tree_hash_cache() - .initialized_slot() - .unwrap(), - start_slot - ); - - // Updating the tree hash cache should be successful despite the long skip. - assert_eq!( - original_cache_state.update_tree_hash_cache().unwrap(), - state.update_tree_hash_cache().unwrap() - ); - - assert_eq!( - original_cache_state - .tree_hash_cache() - .initialized_slot() - .unwrap(), - target_slot - ); } diff --git a/consensus/types/src/beacon_state/tree_hash_cache.rs b/consensus/types/src/beacon_state/tree_hash_cache.rs deleted file mode 100644 index 290020b1b35..00000000000 --- a/consensus/types/src/beacon_state/tree_hash_cache.rs +++ /dev/null @@ -1,645 +0,0 @@ -#![allow(clippy::arithmetic_side_effects)] -#![allow(clippy::disallowed_methods)] -#![allow(clippy::indexing_slicing)] - -use super::Error; -use crate::historical_summary::HistoricalSummaryCache; -use crate::{BeaconState, EthSpec, Hash256, ParticipationList, Slot, Unsigned, Validator}; -use cached_tree_hash::{int_log, CacheArena, CachedTreeHash, TreeHashCache}; -use rayon::prelude::*; -use ssz_derive::{Decode, Encode}; -use ssz_types::VariableList; -use std::cmp::Ordering; -use tree_hash::{mix_in_length, MerkleHasher, TreeHash}; - -/// The number of leaves (including padding) on the `BeaconState` Merkle tree. -/// -/// ## Note -/// -/// This constant is set with the assumption that there are `> 16` and `<= 32` fields on the -/// `BeaconState`. **Tree hashing will fail if this value is set incorrectly.** -pub const NUM_BEACON_STATE_HASH_TREE_ROOT_LEAVES: usize = 32; - -/// The number of nodes in the Merkle tree of a validator record. -const NODES_PER_VALIDATOR: usize = 15; - -/// The number of validator record tree hash caches stored in each arena. -/// -/// This is primarily used for concurrency; if we have 16 validators and set `VALIDATORS_PER_ARENA -/// == 8` then it is possible to do a 2-core concurrent hash. -/// -/// Do not set to 0. -const VALIDATORS_PER_ARENA: usize = 4_096; - -#[derive(Debug, PartialEq, Clone, Encode, Decode)] -pub struct Eth1DataVotesTreeHashCache { - arena: CacheArena, - tree_hash_cache: TreeHashCache, - voting_period: u64, - roots: VariableList, -} - -impl Eth1DataVotesTreeHashCache { - /// Instantiates a new cache. - /// - /// Allocates the necessary memory to store all of the cached Merkle trees. Only the leaves are - /// hashed, leaving the internal nodes as all-zeros. - pub fn new(state: &BeaconState) -> Self { - let mut arena = CacheArena::default(); - let roots: VariableList<_, _> = state - .eth1_data_votes() - .iter() - .map(|eth1_data| eth1_data.tree_hash_root()) - .collect::>() - .into(); - let tree_hash_cache = roots.new_tree_hash_cache(&mut arena); - - Self { - arena, - tree_hash_cache, - voting_period: Self::voting_period(state.slot()), - roots, - } - } - - fn voting_period(slot: Slot) -> u64 { - slot.as_u64() / E::SlotsPerEth1VotingPeriod::to_u64() - } - - pub fn recalculate_tree_hash_root(&mut self, state: &BeaconState) -> Result { - if state.eth1_data_votes().len() < self.roots.len() - || Self::voting_period(state.slot()) != self.voting_period - { - *self = Self::new(state); - } - - state - .eth1_data_votes() - .iter() - .skip(self.roots.len()) - .try_for_each(|eth1_data| self.roots.push(eth1_data.tree_hash_root()))?; - - self.roots - .recalculate_tree_hash_root(&mut self.arena, &mut self.tree_hash_cache) - .map_err(Into::into) - } -} - -/// A cache that performs a caching tree hash of the entire `BeaconState` struct. -/// -/// This type is a wrapper around the inner cache, which does all the work. -#[derive(Debug, Default, PartialEq, Clone)] -pub struct BeaconTreeHashCache { - inner: Option>, -} - -impl BeaconTreeHashCache { - pub fn new(state: &BeaconState) -> Self { - Self { - inner: Some(BeaconTreeHashCacheInner::new(state)), - } - } - - pub fn is_initialized(&self) -> bool { - self.inner.is_some() - } - - /// Move the inner cache out so that the containing `BeaconState` can be borrowed. - pub fn take(&mut self) -> Option> { - self.inner.take() - } - - /// Restore the inner cache after using `take`. - pub fn restore(&mut self, inner: BeaconTreeHashCacheInner) { - self.inner = Some(inner); - } - - /// Make the cache empty. - pub fn uninitialize(&mut self) { - self.inner = None; - } - - /// Return the slot at which the cache was last updated. - /// - /// This should probably only be used during testing. - pub fn initialized_slot(&self) -> Option { - Some(self.inner.as_ref()?.previous_state?.1) - } -} - -#[derive(Debug, PartialEq, Clone)] -pub struct BeaconTreeHashCacheInner { - /// Tracks the previously generated state root to ensure the next state root provided descends - /// directly from this state. - previous_state: Option<(Hash256, Slot)>, - // Validators cache - validators: ValidatorsListTreeHashCache, - // Arenas - fixed_arena: CacheArena, - balances_arena: CacheArena, - slashings_arena: CacheArena, - // Caches - block_roots: TreeHashCache, - state_roots: TreeHashCache, - historical_roots: TreeHashCache, - historical_summaries: OptionalTreeHashCache, - balances: TreeHashCache, - randao_mixes: TreeHashCache, - slashings: TreeHashCache, - eth1_data_votes: Eth1DataVotesTreeHashCache, - inactivity_scores: OptionalTreeHashCache, - // Participation caches - previous_epoch_participation: OptionalTreeHashCache, - current_epoch_participation: OptionalTreeHashCache, -} - -impl BeaconTreeHashCacheInner { - /// Instantiates a new cache. - /// - /// Allocates the necessary memory to store all of the cached Merkle trees. Only the leaves are - /// hashed, leaving the internal nodes as all-zeros. - pub fn new(state: &BeaconState) -> Self { - let mut fixed_arena = CacheArena::default(); - let block_roots = state.block_roots().new_tree_hash_cache(&mut fixed_arena); - let state_roots = state.state_roots().new_tree_hash_cache(&mut fixed_arena); - let historical_roots = state - .historical_roots() - .new_tree_hash_cache(&mut fixed_arena); - let historical_summaries = OptionalTreeHashCache::new( - state - .historical_summaries() - .ok() - .map(HistoricalSummaryCache::new) - .as_ref(), - ); - - let randao_mixes = state.randao_mixes().new_tree_hash_cache(&mut fixed_arena); - - let validators = ValidatorsListTreeHashCache::new::(state.validators()); - - let mut balances_arena = CacheArena::default(); - let balances = state.balances().new_tree_hash_cache(&mut balances_arena); - - let mut slashings_arena = CacheArena::default(); - let slashings = state.slashings().new_tree_hash_cache(&mut slashings_arena); - - let inactivity_scores = OptionalTreeHashCache::new(state.inactivity_scores().ok()); - - let previous_epoch_participation = OptionalTreeHashCache::new( - state - .previous_epoch_participation() - .ok() - .map(ParticipationList::new) - .as_ref(), - ); - let current_epoch_participation = OptionalTreeHashCache::new( - state - .current_epoch_participation() - .ok() - .map(ParticipationList::new) - .as_ref(), - ); - - Self { - previous_state: None, - validators, - fixed_arena, - balances_arena, - slashings_arena, - block_roots, - state_roots, - historical_roots, - historical_summaries, - balances, - randao_mixes, - slashings, - inactivity_scores, - eth1_data_votes: Eth1DataVotesTreeHashCache::new(state), - previous_epoch_participation, - current_epoch_participation, - } - } - - pub fn recalculate_tree_hash_leaves( - &mut self, - state: &BeaconState, - ) -> Result, Error> { - let mut leaves = vec![ - // Genesis data leaves. - state.genesis_time().tree_hash_root(), - state.genesis_validators_root().tree_hash_root(), - // Current fork data leaves. - state.slot().tree_hash_root(), - state.fork().tree_hash_root(), - state.latest_block_header().tree_hash_root(), - // Roots leaves. - state - .block_roots() - .recalculate_tree_hash_root(&mut self.fixed_arena, &mut self.block_roots)?, - state - .state_roots() - .recalculate_tree_hash_root(&mut self.fixed_arena, &mut self.state_roots)?, - state - .historical_roots() - .recalculate_tree_hash_root(&mut self.fixed_arena, &mut self.historical_roots)?, - // Eth1 Data leaves. - state.eth1_data().tree_hash_root(), - self.eth1_data_votes.recalculate_tree_hash_root(state)?, - state.eth1_deposit_index().tree_hash_root(), - // Validator leaves. - self.validators - .recalculate_tree_hash_root(state.validators())?, - state - .balances() - .recalculate_tree_hash_root(&mut self.balances_arena, &mut self.balances)?, - state - .randao_mixes() - .recalculate_tree_hash_root(&mut self.fixed_arena, &mut self.randao_mixes)?, - state - .slashings() - .recalculate_tree_hash_root(&mut self.slashings_arena, &mut self.slashings)?, - ]; - - // Participation - if let BeaconState::Base(state) = state { - leaves.push(state.previous_epoch_attestations.tree_hash_root()); - leaves.push(state.current_epoch_attestations.tree_hash_root()); - } else { - leaves.push( - self.previous_epoch_participation - .recalculate_tree_hash_root(&ParticipationList::new( - state.previous_epoch_participation()?, - ))?, - ); - leaves.push( - self.current_epoch_participation - .recalculate_tree_hash_root(&ParticipationList::new( - state.current_epoch_participation()?, - ))?, - ); - } - // Checkpoint leaves - leaves.push(state.justification_bits().tree_hash_root()); - leaves.push(state.previous_justified_checkpoint().tree_hash_root()); - leaves.push(state.current_justified_checkpoint().tree_hash_root()); - leaves.push(state.finalized_checkpoint().tree_hash_root()); - // Inactivity & light-client sync committees (Altair and later). - if let Ok(inactivity_scores) = state.inactivity_scores() { - leaves.push( - self.inactivity_scores - .recalculate_tree_hash_root(inactivity_scores)?, - ); - } - if let Ok(current_sync_committee) = state.current_sync_committee() { - leaves.push(current_sync_committee.tree_hash_root()); - } - - if let Ok(next_sync_committee) = state.next_sync_committee() { - leaves.push(next_sync_committee.tree_hash_root()); - } - - // Execution payload (merge and later). - if let Ok(payload_header) = state.latest_execution_payload_header() { - leaves.push(payload_header.tree_hash_root()); - } - - // Withdrawal indices (Capella and later). - if let Ok(next_withdrawal_index) = state.next_withdrawal_index() { - leaves.push(next_withdrawal_index.tree_hash_root()); - } - if let Ok(next_withdrawal_validator_index) = state.next_withdrawal_validator_index() { - leaves.push(next_withdrawal_validator_index.tree_hash_root()); - } - - // Historical roots/summaries (Capella and later). - if let Ok(historical_summaries) = state.historical_summaries() { - leaves.push( - self.historical_summaries.recalculate_tree_hash_root( - &HistoricalSummaryCache::new(historical_summaries), - )?, - ); - } - - Ok(leaves) - } - - /// Updates the cache and returns the tree hash root for the given `state`. - /// - /// The provided `state` should be a descendant of the last `state` given to this function, or - /// the `Self::new` function. If the state is more than `SLOTS_PER_HISTORICAL_ROOT` slots - /// after `self.previous_state` then the whole cache will be re-initialized. - pub fn recalculate_tree_hash_root(&mut self, state: &BeaconState) -> Result { - // If this cache has previously produced a root, ensure that it is in the state root - // history of this state. - // - // This ensures that the states applied have a linear history, this - // allows us to make assumptions about how the state changes over times and produce a more - // efficient algorithm. - if let Some((previous_root, previous_slot)) = self.previous_state { - // The previously-hashed state must not be newer than `state`. - if previous_slot > state.slot() { - return Err(Error::TreeHashCacheSkippedSlot { - cache: previous_slot, - state: state.slot(), - }); - } - - // If the state is newer, the previous root must be in the history of the given state. - // If the previous slot is out of range of the `state_roots` array (indicating a long - // gap between the cache's last use and the current state) then we re-initialize. - match state.get_state_root(previous_slot) { - Ok(state_previous_root) if *state_previous_root == previous_root => {} - Ok(_) => return Err(Error::NonLinearTreeHashCacheHistory), - Err(Error::SlotOutOfBounds) => { - *self = Self::new(state); - } - Err(e) => return Err(e), - } - } - - let mut hasher = MerkleHasher::with_leaves(NUM_BEACON_STATE_HASH_TREE_ROOT_LEAVES); - - let leaves = self.recalculate_tree_hash_leaves(state)?; - for leaf in leaves { - hasher.write(leaf.as_bytes())?; - } - - let root = hasher.finish()?; - - self.previous_state = Some((root, state.slot())); - - Ok(root) - } - - /// Updates the cache and provides the root of the given `validators`. - pub fn recalculate_validators_tree_hash_root( - &mut self, - validators: &[Validator], - ) -> Result { - self.validators.recalculate_tree_hash_root(validators) - } -} - -/// A specialized cache for computing the tree hash root of `state.validators`. -#[derive(Debug, PartialEq, Clone, Default, Encode, Decode)] -struct ValidatorsListTreeHashCache { - list_arena: CacheArena, - list_cache: TreeHashCache, - values: ParallelValidatorTreeHash, -} - -impl ValidatorsListTreeHashCache { - /// Instantiates a new cache. - /// - /// Allocates the necessary memory to store all of the cached Merkle trees but does perform any - /// hashing. - fn new(validators: &[Validator]) -> Self { - let mut list_arena = CacheArena::default(); - Self { - list_cache: TreeHashCache::new( - &mut list_arena, - int_log(E::ValidatorRegistryLimit::to_usize()), - validators.len(), - ), - list_arena, - values: ParallelValidatorTreeHash::new(validators), - } - } - - /// Updates the cache and returns the tree hash root for the given `state`. - /// - /// This function makes assumptions that the `validators` list will only change in accordance - /// with valid per-block/per-slot state transitions. - fn recalculate_tree_hash_root(&mut self, validators: &[Validator]) -> Result { - let mut list_arena = std::mem::take(&mut self.list_arena); - - let leaves = self.values.leaves(validators)?; - let num_leaves = leaves.iter().map(|arena| arena.len()).sum(); - - let leaves_iter = ForcedExactSizeIterator { - iter: leaves.into_iter().flatten().map(|h| h.to_fixed_bytes()), - len: num_leaves, - }; - - let list_root = self - .list_cache - .recalculate_merkle_root(&mut list_arena, leaves_iter)?; - - self.list_arena = list_arena; - - Ok(mix_in_length(&list_root, validators.len())) - } -} - -/// Provides a wrapper around some `iter` if the number of items in the iterator is known to the -/// programmer but not the compiler. This allows use of `ExactSizeIterator` in some occasions. -/// -/// Care should be taken to ensure `len` is accurate. -struct ForcedExactSizeIterator { - iter: I, - len: usize, -} - -impl> Iterator for ForcedExactSizeIterator { - type Item = V; - - fn next(&mut self) -> Option { - self.iter.next() - } -} - -impl> ExactSizeIterator for ForcedExactSizeIterator { - fn len(&self) -> usize { - self.len - } -} - -/// Provides a cache for each of the `Validator` objects in `state.validators` and computes the -/// roots of these using Rayon parallelization. -#[derive(Debug, PartialEq, Clone, Default, Encode, Decode)] -pub struct ParallelValidatorTreeHash { - /// Each arena and its associated sub-trees. - arenas: Vec<(CacheArena, Vec)>, -} - -impl ParallelValidatorTreeHash { - /// Instantiates a new cache. - /// - /// Allocates the necessary memory to store all of the cached Merkle trees but does perform any - /// hashing. - fn new(validators: &[Validator]) -> Self { - let num_arenas = std::cmp::max( - 1, - (validators.len() + VALIDATORS_PER_ARENA - 1) / VALIDATORS_PER_ARENA, - ); - - let mut arenas = (1..=num_arenas) - .map(|i| { - let num_validators = if i == num_arenas { - validators.len() % VALIDATORS_PER_ARENA - } else { - VALIDATORS_PER_ARENA - }; - NODES_PER_VALIDATOR * num_validators - }) - .map(|capacity| (CacheArena::with_capacity(capacity), vec![])) - .collect::>(); - - validators.iter().enumerate().for_each(|(i, v)| { - let (arena, caches) = &mut arenas[i / VALIDATORS_PER_ARENA]; - caches.push(v.new_tree_hash_cache(arena)) - }); - - Self { arenas } - } - - /// Returns the number of validators stored in self. - fn len(&self) -> usize { - self.arenas.last().map_or(0, |last| { - // Subtraction cannot underflow because `.last()` ensures the `.len() > 0`. - (self.arenas.len() - 1) * VALIDATORS_PER_ARENA + last.1.len() - }) - } - - /// Updates the caches for each `Validator` in `validators` and returns a list that maps 1:1 - /// with `validators` to the hash of each validator. - /// - /// This function makes assumptions that the `validators` list will only change in accordance - /// with valid per-block/per-slot state transitions. - fn leaves(&mut self, validators: &[Validator]) -> Result>, Error> { - match self.len().cmp(&validators.len()) { - Ordering::Less => validators.iter().skip(self.len()).for_each(|v| { - if self - .arenas - .last() - .map_or(true, |last| last.1.len() >= VALIDATORS_PER_ARENA) - { - let mut arena = CacheArena::default(); - let cache = v.new_tree_hash_cache(&mut arena); - self.arenas.push((arena, vec![cache])) - } else { - let (arena, caches) = &mut self - .arenas - .last_mut() - .expect("Cannot reach this block if arenas is empty."); - caches.push(v.new_tree_hash_cache(arena)) - } - }), - Ordering::Greater => { - return Err(Error::ValidatorRegistryShrunk); - } - Ordering::Equal => (), - } - - self.arenas - .par_iter_mut() - .enumerate() - .map(|(arena_index, (arena, caches))| { - caches - .iter_mut() - .enumerate() - .map(move |(cache_index, cache)| { - let val_index = (arena_index * VALIDATORS_PER_ARENA) + cache_index; - - let validator = validators - .get(val_index) - .ok_or(Error::TreeHashCacheInconsistent)?; - - validator - .recalculate_tree_hash_root(arena, cache) - .map_err(Error::CachedTreeHashError) - }) - .collect() - }) - .collect() - } -} - -#[derive(Debug, PartialEq, Clone)] -pub struct OptionalTreeHashCache { - inner: Option, -} - -#[derive(Debug, PartialEq, Clone)] -pub struct OptionalTreeHashCacheInner { - arena: CacheArena, - tree_hash_cache: TreeHashCache, -} - -impl OptionalTreeHashCache { - /// Initialize a new cache if `item.is_some()`. - fn new>(item: Option<&C>) -> Self { - let inner = item.map(OptionalTreeHashCacheInner::new); - Self { inner } - } - - /// Compute the tree hash root for the given `item`. - /// - /// This function will initialize the inner cache if necessary (e.g. when crossing the fork). - fn recalculate_tree_hash_root>( - &mut self, - item: &C, - ) -> Result { - let cache = self - .inner - .get_or_insert_with(|| OptionalTreeHashCacheInner::new(item)); - item.recalculate_tree_hash_root(&mut cache.arena, &mut cache.tree_hash_cache) - .map_err(Into::into) - } -} - -impl OptionalTreeHashCacheInner { - fn new>(item: &C) -> Self { - let mut arena = CacheArena::default(); - let tree_hash_cache = item.new_tree_hash_cache(&mut arena); - OptionalTreeHashCacheInner { - arena, - tree_hash_cache, - } - } -} - -impl arbitrary::Arbitrary<'_> for BeaconTreeHashCache { - fn arbitrary(_u: &mut arbitrary::Unstructured<'_>) -> arbitrary::Result { - Ok(Self::default()) - } -} - -#[cfg(test)] -mod test { - use super::*; - use crate::{MainnetEthSpec, ParticipationFlags}; - - #[test] - fn validator_node_count() { - let mut arena = CacheArena::default(); - let v = Validator::default(); - let _cache = v.new_tree_hash_cache(&mut arena); - assert_eq!(arena.backing_len(), NODES_PER_VALIDATOR); - } - - #[test] - fn participation_flags() { - type N = ::ValidatorRegistryLimit; - let len = 65; - let mut test_flag = ParticipationFlags::default(); - test_flag.add_flag(0).unwrap(); - let epoch_participation = VariableList::<_, N>::new(vec![test_flag; len]).unwrap(); - - let mut cache = OptionalTreeHashCache { inner: None }; - - let cache_root = cache - .recalculate_tree_hash_root(&ParticipationList::new(&epoch_participation)) - .unwrap(); - let recalc_root = cache - .recalculate_tree_hash_root(&ParticipationList::new(&epoch_participation)) - .unwrap(); - - assert_eq!(cache_root, recalc_root, "recalculated root should match"); - assert_eq!( - cache_root, - epoch_participation.tree_hash_root(), - "cached root should match uncached" - ); - } -} diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 31b1307aa7f..e54bc2f4f97 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -1,7 +1,7 @@ use crate::test_utils::TestRandom; use crate::{ beacon_block_body::BLOB_KZG_COMMITMENTS_INDEX, BeaconBlockHeader, BeaconStateError, Blob, - EthSpec, Hash256, SignedBeaconBlockHeader, Slot, + EthSpec, FixedVector, Hash256, SignedBeaconBlockHeader, Slot, VariableList, }; use crate::{KzgProofs, SignedBeaconBlock}; use bls::Signature; @@ -16,7 +16,6 @@ use safe_arith::{ArithError, SafeArith}; use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; -use ssz_types::{FixedVector, VariableList}; use std::fmt::Debug; use std::hash::Hash; use std::sync::Arc; diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index 3d0b0aca41d..02850304f1d 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -32,7 +32,8 @@ use tree_hash_derive::TreeHash; tree_hash(enum_behaviour = "transparent") ), cast_error(ty = "Error", expr = "BeaconStateError::IncorrectStateVariant"), - partial_getter_error(ty = "Error", expr = "BeaconStateError::IncorrectStateVariant") + partial_getter_error(ty = "Error", expr = "BeaconStateError::IncorrectStateVariant"), + map_ref_into(ExecutionPayloadHeader) )] #[derive( Debug, Clone, Serialize, Deserialize, Encode, TreeHash, Derivative, arbitrary::Arbitrary, @@ -364,6 +365,27 @@ impl TryFrom> for ExecutionPayloadHeaderDe } } +impl<'a, E: EthSpec> ExecutionPayloadHeaderRefMut<'a, E> { + /// Mutate through + pub fn replace(self, header: ExecutionPayloadHeader) -> Result<(), BeaconStateError> { + match self { + ExecutionPayloadHeaderRefMut::Merge(mut_ref) => { + *mut_ref = header.try_into()?; + } + ExecutionPayloadHeaderRefMut::Capella(mut_ref) => { + *mut_ref = header.try_into()?; + } + ExecutionPayloadHeaderRefMut::Deneb(mut_ref) => { + *mut_ref = header.try_into()?; + } + ExecutionPayloadHeaderRefMut::Electra(mut_ref) => { + *mut_ref = header.try_into()?; + } + } + Ok(()) + } +} + impl TryFrom> for ExecutionPayloadHeaderElectra { type Error = BeaconStateError; fn try_from(header: ExecutionPayloadHeader) -> Result { diff --git a/consensus/types/src/historical_batch.rs b/consensus/types/src/historical_batch.rs index 1c565c0092d..7bac9699eb6 100644 --- a/consensus/types/src/historical_batch.rs +++ b/consensus/types/src/historical_batch.rs @@ -23,8 +23,10 @@ use tree_hash_derive::TreeHash; )] #[arbitrary(bound = "E: EthSpec")] pub struct HistoricalBatch { - pub block_roots: FixedVector, - pub state_roots: FixedVector, + #[test_random(default)] + pub block_roots: Vector, + #[test_random(default)] + pub state_roots: Vector, } #[cfg(test)] diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index dee55789398..feefbc48946 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -98,7 +98,6 @@ pub mod sync_committee_contribution; pub mod sync_committee_message; pub mod sync_selection_proof; pub mod sync_subnet_id; -mod tree_hash_impls; pub mod validator_registration_data; pub mod withdrawal; @@ -132,7 +131,7 @@ pub use crate::beacon_block_body::{ }; pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; -pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; +pub use crate::beacon_state::{Error as BeaconStateError, *}; pub use crate::blob_sidecar::{BlobSidecar, BlobSidecarList, BlobsList}; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; @@ -257,8 +256,7 @@ pub use bls::{ AggregatePublicKey, AggregateSignature, Keypair, PublicKey, PublicKeyBytes, SecretKey, Signature, SignatureBytes, }; - pub use kzg::{KzgCommitment, KzgProof, VERSIONED_HASH_VERSION_KZG}; - +pub use milhouse::{self, List, Vector}; pub use ssz_types::{typenum, typenum::Unsigned, BitList, BitVector, FixedVector, VariableList}; pub use superstruct::superstruct; diff --git a/consensus/types/src/light_client_bootstrap.rs b/consensus/types/src/light_client_bootstrap.rs index 33fbf214c80..4d42d357c19 100644 --- a/consensus/types/src/light_client_bootstrap.rs +++ b/consensus/types/src/light_client_bootstrap.rs @@ -1,8 +1,7 @@ -use super::{BeaconState, EthSpec, FixedVector, Hash256, LightClientHeader, SyncCommittee}; use crate::{ - light_client_update::*, test_utils::TestRandom, ChainSpec, ForkName, ForkVersionDeserialize, - LightClientHeaderAltair, LightClientHeaderCapella, LightClientHeaderDeneb, SignedBeaconBlock, - Slot, + light_client_update::*, test_utils::TestRandom, BeaconState, ChainSpec, EthSpec, FixedVector, + ForkName, ForkVersionDeserialize, Hash256, LightClientHeader, LightClientHeaderAltair, + LightClientHeaderCapella, LightClientHeaderDeneb, SignedBeaconBlock, Slot, SyncCommittee, }; use derivative::Derivative; use serde::{Deserialize, Deserializer, Serialize}; diff --git a/consensus/types/src/light_client_finality_update.rs b/consensus/types/src/light_client_finality_update.rs index c5c730e8b83..5f83b0db523 100644 --- a/consensus/types/src/light_client_finality_update.rs +++ b/consensus/types/src/light_client_finality_update.rs @@ -58,6 +58,7 @@ pub struct LightClientFinalityUpdate { #[superstruct(only(Deneb), partial_getter(rename = "finalized_header_deneb"))] pub finalized_header: LightClientHeaderDeneb, /// Merkle proof attesting finalized header. + #[test_random(default)] pub finality_branch: FixedVector, /// current sync aggregate pub sync_aggregate: SyncAggregate, diff --git a/consensus/types/src/light_client_update.rs b/consensus/types/src/light_client_update.rs index af9cbc16610..d5e8cd592df 100644 --- a/consensus/types/src/light_client_update.rs +++ b/consensus/types/src/light_client_update.rs @@ -37,6 +37,7 @@ pub const EXECUTION_PAYLOAD_PROOF_LEN: usize = 4; #[derive(Debug, PartialEq, Clone)] pub enum Error { SszTypesError(ssz_types::Error), + MilhouseError(milhouse::Error), BeaconStateError(beacon_state::Error), ArithError(ArithError), AltairForkNotActive, @@ -65,6 +66,12 @@ impl From for Error { } } +impl From for Error { + fn from(e: milhouse::Error) -> Error { + Error::MilhouseError(e) + } +} + /// A LightClientUpdate is the update we request solely to either complete the bootstrapping process, /// or to sync up to the last committee period, we need to have one ready for each ALTAIR period /// we go over, note: there is no need to keep all of the updates from [ALTAIR_PERIOD, CURRENT_PERIOD]. diff --git a/consensus/types/src/test_utils/test_random.rs b/consensus/types/src/test_utils/test_random.rs index 0adaf81bd7d..72a7a036ccc 100644 --- a/consensus/types/src/test_utils/test_random.rs +++ b/consensus/types/src/test_utils/test_random.rs @@ -87,7 +87,7 @@ where } } -impl TestRandom for FixedVector +impl TestRandom for ssz_types::FixedVector where T: TestRandom, { diff --git a/consensus/types/src/tree_hash_impls.rs b/consensus/types/src/tree_hash_impls.rs deleted file mode 100644 index eb3660d4666..00000000000 --- a/consensus/types/src/tree_hash_impls.rs +++ /dev/null @@ -1,165 +0,0 @@ -//! This module contains custom implementations of `CachedTreeHash` for ETH2-specific types. -//! -//! It makes some assumptions about the layouts and update patterns of other structs in this -//! crate, and should be updated carefully whenever those structs are changed. -use crate::{Epoch, Hash256, PublicKeyBytes, Validator}; -use cached_tree_hash::{int_log, CacheArena, CachedTreeHash, Error, TreeHashCache}; -use int_to_bytes::int_to_fixed_bytes32; -use tree_hash::merkle_root; - -/// Number of struct fields on `Validator`. -const NUM_VALIDATOR_FIELDS: usize = 8; - -impl CachedTreeHash for Validator { - fn new_tree_hash_cache(&self, arena: &mut CacheArena) -> TreeHashCache { - TreeHashCache::new(arena, int_log(NUM_VALIDATOR_FIELDS), NUM_VALIDATOR_FIELDS) - } - - /// Efficiently tree hash a `Validator`, assuming it was updated by a valid state transition. - /// - /// Specifically, we assume that the `pubkey` field is constant. - fn recalculate_tree_hash_root( - &self, - arena: &mut CacheArena, - cache: &mut TreeHashCache, - ) -> Result { - // Otherwise just check the fields which might have changed. - let dirty_indices = cache - .leaves() - .iter_mut(arena)? - .enumerate() - .flat_map(|(i, leaf)| { - // Pubkey field (index 0) is constant. - if i == 0 && cache.initialized { - None - } else if process_field_by_index(self, i, leaf, !cache.initialized) { - Some(i) - } else { - None - } - }) - .collect(); - - cache.update_merkle_root(arena, dirty_indices) - } -} - -fn process_field_by_index( - v: &Validator, - field_idx: usize, - leaf: &mut Hash256, - force_update: bool, -) -> bool { - match field_idx { - 0 => process_pubkey_bytes_field(&v.pubkey, leaf, force_update), - 1 => process_slice_field(v.withdrawal_credentials.as_bytes(), leaf, force_update), - 2 => process_u64_field(v.effective_balance, leaf, force_update), - 3 => process_bool_field(v.slashed, leaf, force_update), - 4 => process_epoch_field(v.activation_eligibility_epoch, leaf, force_update), - 5 => process_epoch_field(v.activation_epoch, leaf, force_update), - 6 => process_epoch_field(v.exit_epoch, leaf, force_update), - 7 => process_epoch_field(v.withdrawable_epoch, leaf, force_update), - _ => panic!( - "Validator type only has {} fields, {} out of bounds", - NUM_VALIDATOR_FIELDS, field_idx - ), - } -} - -fn process_pubkey_bytes_field( - val: &PublicKeyBytes, - leaf: &mut Hash256, - force_update: bool, -) -> bool { - let new_tree_hash = merkle_root(val.as_serialized(), 0); - process_slice_field(new_tree_hash.as_bytes(), leaf, force_update) -} - -fn process_slice_field(new_tree_hash: &[u8], leaf: &mut Hash256, force_update: bool) -> bool { - if force_update || leaf.as_bytes() != new_tree_hash { - leaf.assign_from_slice(new_tree_hash); - true - } else { - false - } -} - -fn process_u64_field(val: u64, leaf: &mut Hash256, force_update: bool) -> bool { - let new_tree_hash = int_to_fixed_bytes32(val); - process_slice_field(&new_tree_hash[..], leaf, force_update) -} - -fn process_epoch_field(val: Epoch, leaf: &mut Hash256, force_update: bool) -> bool { - process_u64_field(val.as_u64(), leaf, force_update) -} - -fn process_bool_field(val: bool, leaf: &mut Hash256, force_update: bool) -> bool { - process_u64_field(val as u64, leaf, force_update) -} - -#[cfg(test)] -mod test { - use super::*; - use crate::test_utils::TestRandom; - use rand::SeedableRng; - use rand_xorshift::XorShiftRng; - use tree_hash::TreeHash; - - fn test_validator_tree_hash(v: &Validator) { - let arena = &mut CacheArena::default(); - - let mut cache = v.new_tree_hash_cache(arena); - // With a fresh cache - assert_eq!( - &v.tree_hash_root()[..], - v.recalculate_tree_hash_root(arena, &mut cache) - .unwrap() - .as_bytes(), - "{:?}", - v - ); - // With a completely up-to-date cache - assert_eq!( - &v.tree_hash_root()[..], - v.recalculate_tree_hash_root(arena, &mut cache) - .unwrap() - .as_bytes(), - "{:?}", - v - ); - } - - #[test] - fn default_validator() { - test_validator_tree_hash(&Validator::default()); - } - - #[test] - fn zeroed_validator() { - let v = Validator { - activation_eligibility_epoch: Epoch::from(0u64), - activation_epoch: Epoch::from(0u64), - ..Default::default() - }; - test_validator_tree_hash(&v); - } - - #[test] - fn random_validators() { - let mut rng = XorShiftRng::from_seed([0xf1; 16]); - let num_validators = 1000; - (0..num_validators) - .map(|_| Validator::random_for_test(&mut rng)) - .for_each(|v| test_validator_tree_hash(&v)); - } - - #[test] - #[allow(clippy::assertions_on_constants)] - pub fn smallvec_size_check() { - // If this test fails we need to go and reassess the length of the `SmallVec` in - // `cached_tree_hash::TreeHashCache`. If the size of the `SmallVec` is too slow we're going - // to start doing heap allocations for each validator, this will fragment memory and slow - // us down. - assert!(NUM_VALIDATOR_FIELDS <= 8,); - } -} diff --git a/lcli/src/new_testnet.rs b/lcli/src/new_testnet.rs index f9da3d2b3e9..edba4249966 100644 --- a/lcli/src/new_testnet.rs +++ b/lcli/src/new_testnet.rs @@ -264,7 +264,7 @@ fn initialize_state_with_validators( let mut state = BeaconState::new(genesis_time, eth1_data, spec); // Seed RANDAO with Eth1 entropy - state.fill_randao_mixes_with(eth1_block_hash); + state.fill_randao_mixes_with(eth1_block_hash).unwrap(); for keypair in keypairs.iter() { let withdrawal_credentials = |pubkey: &PublicKey| { diff --git a/lcli/src/replace_state_pubkeys.rs b/lcli/src/replace_state_pubkeys.rs index 0f9fac3aff9..e8d012b16ec 100644 --- a/lcli/src/replace_state_pubkeys.rs +++ b/lcli/src/replace_state_pubkeys.rs @@ -42,7 +42,8 @@ pub fn run(testnet_dir: PathBuf, matches: &ArgMatches) -> Result<(), let mut deposit_tree = DepositDataTree::create(&[], 0, DEPOSIT_TREE_DEPTH); let mut deposit_root = Hash256::zero(); - for (index, validator) in state.validators_mut().iter_mut().enumerate() { + let validators = state.validators_mut(); + for index in 0..validators.len() { let (secret, _) = recover_validator_secret_from_mnemonic(seed.as_bytes(), index as u32, KeyType::Voting) .map_err(|e| format!("Unable to generate validator key: {:?}", e))?; @@ -52,11 +53,11 @@ pub fn run(testnet_dir: PathBuf, matches: &ArgMatches) -> Result<(), eprintln!("{}: {}", index, keypair.pk); - validator.pubkey = keypair.pk.into(); + validators.get_mut(index).unwrap().pubkey = keypair.pk.into(); // Update the deposit tree. let mut deposit_data = DepositData { - pubkey: validator.pubkey, + pubkey: validators.get(index).unwrap().pubkey, // Set this to a junk value since it's very time consuming to generate the withdrawal // keys and it's not useful for the time being. withdrawal_credentials: Hash256::zero(), diff --git a/lcli/src/skip_slots.rs b/lcli/src/skip_slots.rs index 9e5da7709f1..d421c077d83 100644 --- a/lcli/src/skip_slots.rs +++ b/lcli/src/skip_slots.rs @@ -57,7 +57,7 @@ use std::fs::File; use std::io::prelude::*; use std::path::PathBuf; use std::time::{Duration, Instant}; -use types::{BeaconState, CloneConfig, EthSpec, Hash256}; +use types::{BeaconState, EthSpec, Hash256}; const HTTP_TIMEOUT: Duration = Duration::from_secs(10); @@ -128,7 +128,7 @@ pub fn run( }; for i in 0..runs { - let mut state = state.clone_with(CloneConfig::all()); + let mut state = state.clone(); let start = Instant::now(); diff --git a/lcli/src/transition_blocks.rs b/lcli/src/transition_blocks.rs index c72b41b1d44..77fd352829f 100644 --- a/lcli/src/transition_blocks.rs +++ b/lcli/src/transition_blocks.rs @@ -76,7 +76,7 @@ use ssz::Encode; use state_processing::state_advance::complete_state_advance; use state_processing::{ block_signature_verifier::BlockSignatureVerifier, per_block_processing, AllCaches, - BlockSignatureStrategy, ConsensusContext, StateProcessingStrategy, VerifyBlockRoot, + BlockSignatureStrategy, ConsensusContext, VerifyBlockRoot, }; use std::borrow::Cow; use std::fs::File; @@ -85,7 +85,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::{Duration, Instant}; use store::HotColdDB; -use types::{BeaconState, ChainSpec, CloneConfig, EthSpec, Hash256, SignedBeaconBlock}; +use types::{BeaconState, ChainSpec, EthSpec, Hash256, SignedBeaconBlock}; const HTTP_TIMEOUT: Duration = Duration::from_secs(10); @@ -234,7 +234,7 @@ pub fn run( let mut output_post_state = None; let mut saved_ctxt = None; for i in 0..runs { - let pre_state = pre_state.clone_with(CloneConfig::all()); + let pre_state = pre_state.clone(); let block = block.clone(); let start = Instant::now(); @@ -399,7 +399,6 @@ fn do_transition( &mut pre_state, &block, BlockSignatureStrategy::NoVerification, - StateProcessingStrategy::Accurate, VerifyBlockRoot::True, &mut ctxt, spec, diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index ec10ff4429d..9983717cb1e 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -172,26 +172,6 @@ fn shuffling_cache_set() { .with_config(|config| assert_eq!(config.chain.shuffling_cache_size, 500)); } -#[test] -fn snapshot_cache_default() { - CommandLineTest::new() - .run_with_zero_port() - .with_config(|config| { - assert_eq!( - config.chain.snapshot_cache_size, - beacon_node::beacon_chain::snapshot_cache::DEFAULT_SNAPSHOT_CACHE_SIZE - ) - }); -} - -#[test] -fn snapshot_cache_set() { - CommandLineTest::new() - .flag("state-cache-size", Some("500")) - .run_with_zero_port() - .with_config(|config| assert_eq!(config.chain.snapshot_cache_size, 500)); -} - #[test] fn fork_choice_before_proposal_timeout_default() { CommandLineTest::new() diff --git a/testing/ef_tests/src/case_result.rs b/testing/ef_tests/src/case_result.rs index 67ab9c51bbf..c511d9a1ca0 100644 --- a/testing/ef_tests/src/case_result.rs +++ b/testing/ef_tests/src/case_result.rs @@ -39,6 +39,9 @@ pub fn compare_beacon_state_results_without_caches( if let (Ok(ref mut result), Some(ref mut expected)) = (result.as_mut(), expected.as_mut()) { result.drop_all_caches().unwrap(); expected.drop_all_caches().unwrap(); + + result.apply_pending_mutations().unwrap(); + expected.apply_pending_mutations().unwrap(); } compare_result_detailed(result, expected) diff --git a/testing/ef_tests/src/cases/merkle_proof_validity.rs b/testing/ef_tests/src/cases/merkle_proof_validity.rs index a2e831ade59..cf0b9f77c8f 100644 --- a/testing/ef_tests/src/cases/merkle_proof_validity.rs +++ b/testing/ef_tests/src/cases/merkle_proof_validity.rs @@ -51,7 +51,7 @@ impl LoadCase for MerkleProofValidity { impl Case for MerkleProofValidity { fn result(&self, _case_index: usize, _fork_name: ForkName) -> Result<(), Error> { let mut state = self.state.clone(); - state.initialize_tree_hash_cache(); + state.update_tree_hash_cache().unwrap(); let Ok(proof) = state.compute_merkle_proof(self.merkle_proof.leaf_index) else { return Err(Error::FailedToParseTest( "Could not retrieve merkle proof".to_string(), @@ -77,9 +77,6 @@ impl Case for MerkleProofValidity { } } - // Tree hash cache should still be initialized (not dropped). - assert!(state.tree_hash_cache().is_initialized()); - Ok(()) } } diff --git a/testing/ef_tests/src/cases/sanity_blocks.rs b/testing/ef_tests/src/cases/sanity_blocks.rs index b0902cb5b74..91bb995cc43 100644 --- a/testing/ef_tests/src/cases/sanity_blocks.rs +++ b/testing/ef_tests/src/cases/sanity_blocks.rs @@ -5,7 +5,7 @@ use crate::decode::{ssz_decode_file_with, ssz_decode_state, yaml_decode_file}; use serde::Deserialize; use state_processing::{ per_block_processing, per_slot_processing, BlockProcessingError, BlockSignatureStrategy, - ConsensusContext, StateProcessingStrategy, VerifyBlockRoot, + ConsensusContext, VerifyBlockRoot, }; use types::{BeaconState, RelativeEpoch, SignedBeaconBlock}; @@ -96,7 +96,6 @@ impl Case for SanityBlocks { &mut indiv_state, signed_block, BlockSignatureStrategy::VerifyIndividual, - StateProcessingStrategy::Accurate, VerifyBlockRoot::True, &mut ctxt, spec, @@ -107,7 +106,6 @@ impl Case for SanityBlocks { &mut bulk_state, signed_block, BlockSignatureStrategy::VerifyBulk, - StateProcessingStrategy::Accurate, VerifyBlockRoot::True, &mut ctxt, spec, diff --git a/testing/ef_tests/src/cases/ssz_generic.rs b/testing/ef_tests/src/cases/ssz_generic.rs index bb2465aae10..8de3e217f00 100644 --- a/testing/ef_tests/src/cases/ssz_generic.rs +++ b/testing/ef_tests/src/cases/ssz_generic.rs @@ -1,14 +1,14 @@ #![allow(non_snake_case)] use super::*; -use crate::cases::common::{TestU128, TestU256}; -use crate::decode::{snappy_decode_file, yaml_decode_file}; -use serde::Deserialize; -use serde::{de::Error as SerdeError, Deserializer}; +use crate::cases::common::{SszStaticType, TestU128, TestU256}; +use crate::cases::ssz_static::{check_serialization, check_tree_hash}; +use crate::decode::{log_file_access, snappy_decode_file, yaml_decode_file}; +use serde::{de::Error as SerdeError, Deserialize, Deserializer}; use ssz_derive::{Decode, Encode}; use tree_hash_derive::TreeHash; use types::typenum::*; -use types::{BitList, BitVector, FixedVector, VariableList}; +use types::{BitList, BitVector, FixedVector, ForkName, VariableList, Vector}; #[derive(Debug, Clone, Deserialize)] struct Metadata { @@ -125,6 +125,20 @@ impl Case for SszGeneric { let elem_ty = parts[1]; let length = parts[2]; + // Skip length 0 tests. Milhouse doesn't have any checks against 0-capacity lists. + if length == "0" { + log_file_access(self.path.join("serialized.ssz_snappy")); + return Ok(()); + } + + type_dispatch!( + ssz_generic_test, + (&self.path), + Vector, + <>, + [elem_ty => primitive_type] + [length => typenum] + )?; type_dispatch!( ssz_generic_test, (&self.path), @@ -263,8 +277,8 @@ struct ComplexTestStruct { #[serde(deserialize_with = "byte_list_from_hex_str")] D: VariableList, E: VarTestStruct, - F: FixedVector, - G: FixedVector, + F: Vector, + G: Vector, } #[derive(Debug, Clone, PartialEq, Decode, Encode, TreeHash, Deserialize)] diff --git a/testing/ef_tests/src/cases/ssz_static.rs b/testing/ef_tests/src/cases/ssz_static.rs index e41c90c6e03..5f0ac3525c4 100644 --- a/testing/ef_tests/src/cases/ssz_static.rs +++ b/testing/ef_tests/src/cases/ssz_static.rs @@ -118,7 +118,6 @@ impl Case for SszStaticTHC> { check_tree_hash(&self.roots.root, self.value.tree_hash_root().as_bytes())?; let mut state = self.value.clone(); - state.initialize_tree_hash_cache(); let cached_tree_hash_root = state.update_tree_hash_cache().unwrap(); check_tree_hash(&self.roots.root, cached_tree_hash_root.as_bytes())?; diff --git a/testing/ef_tests/src/cases/transition.rs b/testing/ef_tests/src/cases/transition.rs index 927589948a2..b2c49a96feb 100644 --- a/testing/ef_tests/src/cases/transition.rs +++ b/testing/ef_tests/src/cases/transition.rs @@ -4,7 +4,7 @@ use crate::decode::{ssz_decode_file_with, ssz_decode_state, yaml_decode_file}; use serde::Deserialize; use state_processing::{ per_block_processing, state_advance::complete_state_advance, BlockSignatureStrategy, - ConsensusContext, StateProcessingStrategy, VerifyBlockRoot, + ConsensusContext, VerifyBlockRoot, }; use std::str::FromStr; use types::{BeaconState, Epoch, SignedBeaconBlock}; @@ -114,7 +114,6 @@ impl Case for TransitionTest { &mut state, block, BlockSignatureStrategy::VerifyBulk, - StateProcessingStrategy::Accurate, VerifyBlockRoot::True, &mut ctxt, spec, diff --git a/testing/state_transition_vectors/src/exit.rs b/testing/state_transition_vectors/src/exit.rs index e3cd346da13..61cae6dbe1b 100644 --- a/testing/state_transition_vectors/src/exit.rs +++ b/testing/state_transition_vectors/src/exit.rs @@ -1,7 +1,7 @@ use super::*; use state_processing::{ per_block_processing, per_block_processing::errors::ExitInvalid, BlockProcessingError, - BlockSignatureStrategy, ConsensusContext, StateProcessingStrategy, VerifyBlockRoot, + BlockSignatureStrategy, ConsensusContext, VerifyBlockRoot, }; use types::{BeaconBlock, Epoch}; @@ -68,7 +68,6 @@ impl ExitTest { state, block, BlockSignatureStrategy::VerifyIndividual, - StateProcessingStrategy::Accurate, VerifyBlockRoot::True, &mut ctxt, &E::default_spec(), @@ -332,7 +331,7 @@ mod custom_tests { fn assert_exited(state: &BeaconState, validator_index: usize) { let spec = E::default_spec(); - let validator = &state.validators()[validator_index]; + let validator = &state.validators().get(validator_index).unwrap(); assert_eq!( validator.exit_epoch, // This is correct until we exceed the churn limit. If that happens, we From c4a2bcb9c79828fcbb3c204306c986d5b2e5a246 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 24 Apr 2024 02:02:10 -0400 Subject: [PATCH 25/37] Yaml rust2 (#5635) * use yaml-rust2 --- Cargo.lock | 424 +++--------------------------- consensus/int_to_bytes/Cargo.toml | 2 +- consensus/int_to_bytes/src/lib.rs | 10 +- 3 files changed, 43 insertions(+), 393 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d63773d9a94..bee644e2d79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,7 +18,7 @@ version = "0.3.5" dependencies = [ "account_utils", "bls", - "clap 2.34.0", + "clap", "clap_utils", "directory", "environment", @@ -292,54 +292,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "anstream" -version = "0.6.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" -dependencies = [ - "anstyle", - "anstyle-parse", - "anstyle-query", - "anstyle-wincon", - "colorchoice", - "utf8parse", -] - -[[package]] -name = "anstyle" -version = "1.0.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" - -[[package]] -name = "anstyle-parse" -version = "0.2.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" -dependencies = [ - "utf8parse", -] - -[[package]] -name = "anstyle-query" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" -dependencies = [ - "windows-sys 0.52.0", -] - -[[package]] -name = "anstyle-wincon" -version = "3.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" -dependencies = [ - "anstyle", - "windows-sys 0.52.0", -] - [[package]] name = "anyhow" version = "1.0.82" @@ -494,6 +446,12 @@ dependencies = [ "rand", ] +[[package]] +name = "arraydeque" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236" + [[package]] name = "arrayref" version = "0.3.7" @@ -856,7 +814,7 @@ name = "beacon_node" version = "5.1.3" dependencies = [ "beacon_chain", - "clap 2.34.0", + "clap", "clap_utils", "client", "directory", @@ -1069,7 +1027,7 @@ name = "boot_node" version = "5.1.3" dependencies = [ "beacon_node", - "clap 2.34.0", + "clap", "clap_utils", "eth2_network_config", "ethereum_ssz", @@ -1345,38 +1303,11 @@ dependencies = [ "vec_map", ] -[[package]] -name = "clap" -version = "4.5.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bc066a67923782aa8515dbaea16946c5bcc5addbd668bb80af688e53e548a0" -dependencies = [ - "clap_builder", -] - -[[package]] -name = "clap_builder" -version = "4.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae129e2e766ae0ec03484e609954119f123cc1fe650337e155d03b022f24f7b4" -dependencies = [ - "anstream", - "anstyle", - "clap_lex", - "strsim 0.11.1", -] - -[[package]] -name = "clap_lex" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" - [[package]] name = "clap_utils" version = "0.1.0" dependencies = [ - "clap 2.34.0", + "clap", "dirs", "eth2_network_config", "ethereum-types 0.14.1", @@ -1442,12 +1373,6 @@ dependencies = [ "cc", ] -[[package]] -name = "colorchoice" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" - [[package]] name = "compare_fields" version = "0.2.0" @@ -1473,19 +1398,6 @@ dependencies = [ "crossbeam-utils", ] -[[package]] -name = "console" -version = "0.15.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e1f83fc076bd6dd27517eacdf25fef6c4dfe5f1d7448bafaaf3a26f13b5e4eb" -dependencies = [ - "encode_unicode", - "lazy_static", - "libc", - "unicode-width", - "windows-sys 0.52.0", -] - [[package]] name = "const-hex" version = "1.11.3" @@ -1568,7 +1480,7 @@ checksum = "b01d6de93b2b6c65e17c634a26653a29d107b3c98c607c765bf38d041531cd8f" dependencies = [ "atty", "cast", - "clap 2.34.0", + "clap", "criterion-plot", "csv", "itertools", @@ -1783,18 +1695,8 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" dependencies = [ - "darling_core 0.13.4", - "darling_macro 0.13.4", -] - -[[package]] -name = "darling" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" -dependencies = [ - "darling_core 0.14.4", - "darling_macro 0.14.4", + "darling_core", + "darling_macro", ] [[package]] @@ -1811,38 +1713,13 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "darling_core" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim 0.10.0", - "syn 1.0.109", -] - [[package]] name = "darling_macro" version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" dependencies = [ - "darling_core 0.13.4", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "darling_macro" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" -dependencies = [ - "darling_core 0.14.4", + "darling_core", "quote", "syn 1.0.109", ] @@ -1905,7 +1782,7 @@ version = "0.1.0" dependencies = [ "beacon_chain", "beacon_node", - "clap 2.34.0", + "clap", "clap_utils", "environment", "hex", @@ -2014,37 +1891,6 @@ dependencies = [ "syn 2.0.60", ] -[[package]] -name = "derive_builder" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8" -dependencies = [ - "derive_builder_macro", -] - -[[package]] -name = "derive_builder_core" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f" -dependencies = [ - "darling 0.14.4", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "derive_builder_macro" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e" -dependencies = [ - "derive_builder_core", - "syn 1.0.109", -] - [[package]] name = "derive_more" version = "0.99.17" @@ -2058,19 +1904,6 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "dialoguer" -version = "0.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "658bce805d770f407bc62102fca7c2c64ceef2fbcb2b8bd19d2765ce093980de" -dependencies = [ - "console", - "shell-words", - "tempfile", - "thiserror", - "zeroize", -] - [[package]] name = "diesel" version = "2.1.6" @@ -2142,7 +1975,7 @@ dependencies = [ name = "directory" version = "0.1.0" dependencies = [ - "clap 2.34.0", + "clap", "clap_utils", "eth2_network_config", ] @@ -2236,29 +2069,6 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" -[[package]] -name = "dtt" -version = "0.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6b2dd9ee2d76888dc4c17d6da74629fa11b3cb1e8094fdc159b7f8ff259fc88" -dependencies = [ - "regex", - "serde", - "time", -] - -[[package]] -name = "duct" -version = "0.13.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4ab5718d1224b63252cd0c6f74f6480f9ffeb117438a2e0f5cf6d9a4798929c" -dependencies = [ - "libc", - "once_cell", - "os_pipe", - "shared_child", -] - [[package]] name = "dunce" version = "1.0.4" @@ -2396,12 +2206,6 @@ dependencies = [ "zeroize", ] -[[package]] -name = "encode_unicode" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a357d28ed41a50f9c765dbfe56cbc04a64e53e5fc58ba79fbc34c10ef3df831f" - [[package]] name = "encoding_rs" version = "0.8.34" @@ -2442,16 +2246,6 @@ dependencies = [ "syn 2.0.60", ] -[[package]] -name = "env_filter" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a009aa4810eb158359dda09d0c87378e4bbb89b5a801f016885a4707ba24f7ea" -dependencies = [ - "log", - "regex", -] - [[package]] name = "env_logger" version = "0.8.4" @@ -2475,19 +2269,6 @@ dependencies = [ "termcolor", ] -[[package]] -name = "env_logger" -version = "0.11.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b35839ba51819680ba087cd351788c9a3c476841207e0b8cee0b04722343b9" -dependencies = [ - "anstream", - "anstyle", - "env_filter", - "humantime", - "log", -] - [[package]] name = "environment" version = "0.1.2" @@ -2675,7 +2456,7 @@ dependencies = [ "sha2 0.9.9", "tempfile", "unicode-normalization", - "uuid 0.8.2", + "uuid", "zeroize", ] @@ -2715,7 +2496,7 @@ dependencies = [ "serde_repr", "tempfile", "tiny-bip39", - "uuid 0.8.2", + "uuid", ] [[package]] @@ -2859,7 +2640,7 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6085d7fd3cf84bd2b8fec150d54c8467fb491d8db9c460607c5534f653a0ee38" dependencies = [ - "darling 0.13.4", + "darling", "proc-macro2", "quote", "syn 1.0.109", @@ -3172,12 +2953,6 @@ dependencies = [ "rustc_version 0.4.0", ] -[[package]] -name = "figlet-rs" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4742a071cd9694fc86f9fa1a08fa3e53d40cc899d7ee532295da2d085639fbc5" - [[package]] name = "filesystem" version = "0.1.0" @@ -3283,12 +3058,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "fs_extra" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" - [[package]] name = "funty" version = "1.1.0" @@ -4286,7 +4055,7 @@ version = "0.2.0" dependencies = [ "bytes", "hex", - "serde_yml", + "yaml-rust2", ] [[package]] @@ -4510,7 +4279,7 @@ dependencies = [ "account_utils", "beacon_chain", "bls", - "clap 2.34.0", + "clap", "clap_utils", "deposit_contract", "directory", @@ -5088,7 +4857,7 @@ dependencies = [ "beacon_processor", "bls", "boot_node", - "clap 2.34.0", + "clap", "clap_utils", "database_manager", "directory", @@ -5436,7 +5205,7 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37cb4045d5677b7da537f8cb5d0730d5b6414e3cc81c61e4b50e1f0cbdc73909" dependencies = [ - "darling 0.13.4", + "darling", "itertools", "proc-macro2", "quote", @@ -6044,16 +5813,6 @@ dependencies = [ "types", ] -[[package]] -name = "os_pipe" -version = "1.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57119c3b893986491ec9aa85056780d3a0f3cf4da7cc09dd3650dbd6c6738fb9" -dependencies = [ - "libc", - "windows-sys 0.52.0", -] - [[package]] name = "overload" version = "0.1.1" @@ -7086,30 +6845,6 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" -[[package]] -name = "rlg" -version = "0.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6ccf670238310d5c31a52fed1a3314620d037a64f1e5fbdc71b2c50909134dc" -dependencies = [ - "dtt", - "tokio", - "vrd 0.0.4", -] - -[[package]] -name = "rlg" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e02c717e23f67b23032a4acb01cf63534d6259938d592e6d2451c02f09fc368" -dependencies = [ - "dtt", - "hostname", - "serde_json", - "tokio", - "vrd 0.0.5", -] - [[package]] name = "rlp" version = "0.5.2" @@ -7687,7 +7422,7 @@ version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e182d6ec6f05393cc0e5ed1bf81ad6db3a8feedf8ee515ecdd369809bcce8082" dependencies = [ - "darling 0.13.4", + "darling", "proc-macro2", "quote", "syn 1.0.109", @@ -7706,27 +7441,6 @@ dependencies = [ "unsafe-libyaml", ] -[[package]] -name = "serde_yml" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "196c3750ff0411738366b0d9534ca55fc74d04a3d4284a450039de950bd11938" -dependencies = [ - "dtt", - "env_logger 0.11.3", - "figlet-rs", - "indexmap 2.2.6", - "itoa", - "log", - "openssl", - "rlg 0.0.3", - "ryu", - "serde", - "unsafe-libyaml", - "uuid 1.8.0", - "xtasks", -] - [[package]] name = "sha1" version = "0.10.6" @@ -7803,22 +7517,6 @@ dependencies = [ "lazy_static", ] -[[package]] -name = "shared_child" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0d94659ad3c2137fef23ae75b03d5241d633f8acded53d672decfa0e6e0caef" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "shell-words" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24188a676b6ae68c3b2cb3a01be17fbf7240ce009799bb56d5b1409051e78fde" - [[package]] name = "shlex" version = "1.3.0" @@ -7870,7 +7568,7 @@ dependencies = [ name = "simulator" version = "0.2.0" dependencies = [ - "clap 2.34.0", + "clap", "env_logger 0.9.3", "eth1", "eth2_network_config", @@ -8282,12 +7980,6 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" -[[package]] -name = "strsim" -version = "0.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" - [[package]] name = "strum" version = "0.24.1" @@ -8322,7 +8014,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6f4e1f478a7728f8855d7e620e9a152cf8932c6614f86564c886f9b8141f3201" dependencies = [ - "darling 0.13.4", + "darling", "itertools", "proc-macro2", "quote", @@ -8691,7 +8383,6 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", "socket2 0.5.6", @@ -9022,7 +8713,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ce7bccc538359a213436af7bc95804bdbf1c2a21d80e22953cbe9e096837ff1" dependencies = [ - "darling 0.13.4", + "darling", "quote", "syn 1.0.109", ] @@ -9270,12 +8961,6 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "utf8parse" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" - [[package]] name = "uuid" version = "0.8.2" @@ -9286,15 +8971,6 @@ dependencies = [ "serde", ] -[[package]] -name = "uuid" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a183cf7feeba97b4dd1c0d46788634f6221d87fa961b305bed08c851829efcc0" -dependencies = [ - "getrandom", -] - [[package]] name = "validator_client" version = "0.3.5" @@ -9302,7 +8978,7 @@ dependencies = [ "account_utils", "bincode", "bls", - "clap 2.34.0", + "clap", "clap_utils", "deposit_contract", "directory", @@ -9374,7 +9050,7 @@ version = "0.1.0" dependencies = [ "account_utils", "bls", - "clap 2.34.0", + "clap", "clap_utils", "environment", "eth2", @@ -9423,25 +9099,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" -[[package]] -name = "vrd" -version = "0.0.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a81b8b5b404f3d7afa1b8142a6bc980c20cd68556c634c3db517871aa0402521" -dependencies = [ - "rand", -] - -[[package]] -name = "vrd" -version = "0.0.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee1067b8d17481f5be71b59d11c329e955ffe36348907e0a4a41b619682bb4af" -dependencies = [ - "rand", - "serde", -] - [[package]] name = "wait-timeout" version = "0.2.0" @@ -9634,7 +9291,7 @@ dependencies = [ "beacon_node", "bls", "byteorder", - "clap 2.34.0", + "clap", "diesel", "diesel_migrations", "env_logger 0.9.3", @@ -10099,23 +9756,14 @@ dependencies = [ ] [[package]] -name = "xtasks" -version = "0.0.2" +name = "yaml-rust2" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940db5674e301470e6cd91098b2c68a1fad751a1623575d1133f7456146e6d2f" +checksum = "498f4d102a79ea1c9d4dd27573c0fc96ad74c023e8da38484e47883076da25fb" dependencies = [ - "anyhow", - "clap 4.5.4", - "derive_builder", - "dialoguer", - "dtt", - "duct", - "fs_extra", - "glob", - "rlg 0.0.2", - "serde", - "serde_json", - "vrd 0.0.5", + "arraydeque", + "encoding_rs", + "hashlink", ] [[package]] diff --git a/consensus/int_to_bytes/Cargo.toml b/consensus/int_to_bytes/Cargo.toml index 1958fac6e56..e99d1af8e56 100644 --- a/consensus/int_to_bytes/Cargo.toml +++ b/consensus/int_to_bytes/Cargo.toml @@ -8,5 +8,5 @@ edition = { workspace = true } bytes = { workspace = true } [dev-dependencies] +yaml-rust2 = "0.8" hex = { workspace = true } -serde_yml = "0.0.4" diff --git a/consensus/int_to_bytes/src/lib.rs b/consensus/int_to_bytes/src/lib.rs index 076e7154647..7a1fff7ccd3 100644 --- a/consensus/int_to_bytes/src/lib.rs +++ b/consensus/int_to_bytes/src/lib.rs @@ -78,7 +78,8 @@ pub fn int_to_bytes96(int: u64) -> Vec { #[cfg(test)] mod tests { use super::*; - use std::{collections::HashMap, fs::File, io::prelude::*, path::PathBuf}; + use std::{fs::File, io::prelude::*, path::PathBuf}; + use yaml_rust2::yaml; #[test] fn fixed_bytes32() { @@ -110,13 +111,14 @@ mod tests { file.read_to_string(&mut yaml_str).unwrap(); - let docs: HashMap = serde_yml::from_str(&yaml_str).unwrap(); - let test_cases = docs["test_cases"].as_sequence().unwrap(); + let docs = yaml::YamlLoader::load_from_str(&yaml_str).unwrap(); + let doc = &docs[0]; + let test_cases = doc["test_cases"].as_vec().unwrap(); for test_case in test_cases { let byte_length = test_case["byte_length"].as_i64().unwrap() as u64; let int = test_case["int"].as_i64().unwrap() as u64; - let bytes_string = test_case["bytes"].as_str().unwrap(); + let bytes_string = test_case["bytes"].clone().into_string().unwrap(); let bytes = hex::decode(bytes_string.replace("0x", "")).unwrap(); match byte_length { From 1eaaa4a8bd15f505d37a43a4a11333ea2a64c93f Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 24 Apr 2024 16:02:42 +1000 Subject: [PATCH 26/37] Bump jobserver and fix non-portable builds (#5641) * Bump jobserver and fix non-portable builds --- Cargo.lock | 5 ++--- Cargo.toml | 4 +++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bee644e2d79..335491adc1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1133,7 +1133,6 @@ dependencies = [ "glob", "hex", "libc", - "serde", ] [[package]] @@ -4155,9 +4154,9 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.30" +version = "0.1.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "685a7d121ee3f65ae4fddd72b25a04bb36b6af81bc0828f7d5434c0fe60fa3a2" +checksum = "d2b099aaa34a9751c5bf0878add70444e1ed2dd73f347be99003d4577277de6e" dependencies = [ "libc", ] diff --git a/Cargo.toml b/Cargo.toml index 296dadc1f66..be2011ba286 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -102,7 +102,9 @@ bincode = "1" bitvec = "1" byteorder = "1" bytes = "1" -c-kzg = "1" +# Turn off c-kzg's default features which include `blst/portable`. We can turn on blst's portable +# feature ourselves when desired. +c-kzg = { version = "1", default-features = false } clap = "2" compare_fields_derive = { path = "common/compare_fields_derive" } criterion = "0.3" From c38b05d640bfddc311e44e37cb06962fe5c9c79d Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Wed, 24 Apr 2024 15:02:45 +0900 Subject: [PATCH 27/37] Drop lookup type trait for a simple arg (#5620) * Drop lookup type trait for a simple arg --- .../network/src/sync/block_lookups/common.rs | 63 +++---- .../network/src/sync/block_lookups/mod.rs | 40 ++-- .../src/sync/block_lookups/parent_lookup.rs | 7 +- .../sync/block_lookups/single_block_lookup.rs | 172 ++---------------- beacon_node/network/src/sync/manager.rs | 21 +-- 5 files changed, 73 insertions(+), 230 deletions(-) diff --git a/beacon_node/network/src/sync/block_lookups/common.rs b/beacon_node/network/src/sync/block_lookups/common.rs index 3bd39301b21..7193dd6e216 100644 --- a/beacon_node/network/src/sync/block_lookups/common.rs +++ b/beacon_node/network/src/sync/block_lookups/common.rs @@ -29,34 +29,12 @@ pub enum LookupType { Parent, } -/// This trait helps differentiate `SingleBlockLookup`s from `ParentLookup`s .This is useful in -/// ensuring requests and responses are handled separately and enables us to use different failure -/// tolerances for each, while re-using the same basic request and retry logic. -pub trait Lookup { - const MAX_ATTEMPTS: u8; - fn lookup_type() -> LookupType; - fn max_attempts() -> u8 { - Self::MAX_ATTEMPTS - } -} - -/// A `Lookup` that is a part of a `ParentLookup`. -pub struct Parent; - -impl Lookup for Parent { - const MAX_ATTEMPTS: u8 = PARENT_FAIL_TOLERANCE; - fn lookup_type() -> LookupType { - LookupType::Parent - } -} - -/// A `Lookup` that part of a single block lookup. -pub struct Current; - -impl Lookup for Current { - const MAX_ATTEMPTS: u8 = SINGLE_BLOCK_LOOKUP_MAX_ATTEMPTS; - fn lookup_type() -> LookupType { - LookupType::Current +impl LookupType { + fn max_attempts(&self) -> u8 { + match self { + LookupType::Current => SINGLE_BLOCK_LOOKUP_MAX_ATTEMPTS, + LookupType::Parent => PARENT_FAIL_TOLERANCE, + } } } @@ -68,7 +46,7 @@ impl Lookup for Current { /// The use of the `ResponseType` associated type gives us a degree of type /// safety when handling a block/blob response ensuring we only mutate the correct corresponding /// state. -pub trait RequestState { +pub trait RequestState { /// The type of the request . type RequestType; @@ -81,9 +59,12 @@ pub trait RequestState { /* Request building methods */ /// Construct a new request. - fn build_request(&mut self) -> Result<(PeerId, Self::RequestType), LookupRequestError> { + fn build_request( + &mut self, + lookup_type: LookupType, + ) -> Result<(PeerId, Self::RequestType), LookupRequestError> { // Verify and construct request. - self.too_many_attempts()?; + self.too_many_attempts(lookup_type)?; let peer = self.get_peer()?; let request = self.new_request(); Ok((peer, request)) @@ -93,6 +74,7 @@ pub trait RequestState { fn build_request_and_send( &mut self, id: Id, + lookup_type: LookupType, cx: &mut SyncNetworkContext, ) -> Result<(), LookupRequestError> { // Check if request is necessary. @@ -101,7 +83,7 @@ pub trait RequestState { } // Construct request. - let (peer_id, request) = self.build_request()?; + let (peer_id, request) = self.build_request(lookup_type)?; // Update request state. let req_counter = self.get_state_mut().on_download_start(peer_id); @@ -110,17 +92,16 @@ pub trait RequestState { let id = SingleLookupReqId { id, req_counter, - lookup_type: L::lookup_type(), + lookup_type, }; Self::make_request(id, peer_id, request, cx) } /// Verify the current request has not exceeded the maximum number of attempts. - fn too_many_attempts(&self) -> Result<(), LookupRequestError> { - let max_attempts = L::max_attempts(); + fn too_many_attempts(&self, lookup_type: LookupType) -> Result<(), LookupRequestError> { let request_state = self.get_state(); - if request_state.failed_attempts() >= max_attempts { + if request_state.failed_attempts() >= lookup_type.max_attempts() { let cannot_process = request_state.more_failed_processing_attempts(); Err(LookupRequestError::TooManyAttempts { cannot_process }) } else { @@ -187,7 +168,7 @@ pub trait RequestState { fn response_type() -> ResponseType; /// A getter for the `BlockRequestState` or `BlobRequestState` associated with this trait. - fn request_state_mut(request: &mut SingleBlockLookup) -> &mut Self; + fn request_state_mut(request: &mut SingleBlockLookup) -> &mut Self; /// A getter for a reference to the `SingleLookupRequestState` associated with this trait. fn get_state(&self) -> &SingleLookupRequestState; @@ -196,7 +177,7 @@ pub trait RequestState { fn get_state_mut(&mut self) -> &mut SingleLookupRequestState; } -impl RequestState for BlockRequestState { +impl RequestState for BlockRequestState { type RequestType = BlocksByRootSingleRequest; type VerifiedResponseType = Arc>; type ReconstructedResponseType = RpcBlock; @@ -253,7 +234,7 @@ impl RequestState for BlockRequestState fn response_type() -> ResponseType { ResponseType::Block } - fn request_state_mut(request: &mut SingleBlockLookup) -> &mut Self { + fn request_state_mut(request: &mut SingleBlockLookup) -> &mut Self { &mut request.block_request_state } fn get_state(&self) -> &SingleLookupRequestState { @@ -264,7 +245,7 @@ impl RequestState for BlockRequestState } } -impl RequestState for BlobRequestState { +impl RequestState for BlobRequestState { type RequestType = BlobsByRootSingleBlockRequest; type VerifiedResponseType = FixedBlobSidecarList; type ReconstructedResponseType = FixedBlobSidecarList; @@ -328,7 +309,7 @@ impl RequestState for BlobRequestState ResponseType { ResponseType::Blob } - fn request_state_mut(request: &mut SingleBlockLookup) -> &mut Self { + fn request_state_mut(request: &mut SingleBlockLookup) -> &mut Self { &mut request.blob_request_state } fn get_state(&self) -> &SingleLookupRequestState { diff --git a/beacon_node/network/src/sync/block_lookups/mod.rs b/beacon_node/network/src/sync/block_lookups/mod.rs index fa2683fb0f0..a2909b49dd1 100644 --- a/beacon_node/network/src/sync/block_lookups/mod.rs +++ b/beacon_node/network/src/sync/block_lookups/mod.rs @@ -16,9 +16,6 @@ use beacon_chain::data_availability_checker::{ }; use beacon_chain::validator_monitor::timestamp_now; use beacon_chain::{AvailabilityProcessingStatus, BeaconChainTypes, BlockError}; -pub use common::Current; -pub use common::Lookup; -pub use common::Parent; pub use common::RequestState; use fnv::FnvHashMap; use lighthouse_network::{PeerAction, PeerId}; @@ -55,12 +52,12 @@ pub struct BlockLookups { /// Parent chain lookups being downloaded. parent_lookups: SmallVec<[ParentLookup; 3]>, - processing_parent_lookups: HashMap, SingleBlockLookup)>, + processing_parent_lookups: HashMap, SingleBlockLookup)>, /// A cache of failed chain lookups to prevent duplicate searches. failed_chains: LRUTimeCache, - single_block_lookups: FnvHashMap>, + single_block_lookups: FnvHashMap>, pub(crate) da_checker: Arc>, @@ -131,7 +128,7 @@ impl BlockLookups { /// Attempts to trigger the request matching the given `block_root`. pub fn trigger_single_lookup( &mut self, - mut single_block_lookup: SingleBlockLookup, + mut single_block_lookup: SingleBlockLookup, cx: &mut SyncNetworkContext, ) { let block_root = single_block_lookup.block_root(); @@ -147,7 +144,7 @@ impl BlockLookups { } /// Adds a lookup to the `single_block_lookups` map. - pub fn add_single_lookup(&mut self, single_block_lookup: SingleBlockLookup) { + pub fn add_single_lookup(&mut self, single_block_lookup: SingleBlockLookup) { self.single_block_lookups .insert(single_block_lookup.id, single_block_lookup); @@ -212,6 +209,7 @@ impl BlockLookups { peers, self.da_checker.clone(), cx.next_id(), + LookupType::Current, ); debug!( @@ -284,10 +282,10 @@ impl BlockLookups { /// Get a single block lookup by its ID. This method additionally ensures the `req_counter` /// matches the current `req_counter` for the lookup. This ensures any stale responses from requests /// that have been retried are ignored. - fn get_single_lookup>( + fn get_single_lookup>( &mut self, id: SingleLookupReqId, - ) -> Option> { + ) -> Option> { let mut lookup = self.single_block_lookups.remove(&id.id)?; let request_state = R::request_state_mut(&mut lookup); @@ -314,7 +312,7 @@ impl BlockLookups { } /// Process a block or blob response received from a single lookup request. - pub fn single_lookup_response>( + pub fn single_lookup_response>( &mut self, lookup_id: SingleLookupReqId, peer_id: PeerId, @@ -345,7 +343,7 @@ impl BlockLookups { "response_type" => ?response_type, ); - match self.handle_verified_response::( + match self.handle_verified_response::( seen_timestamp, cx, BlockProcessType::SingleBlock { id: lookup.id }, @@ -372,13 +370,13 @@ impl BlockLookups { /// Consolidates error handling for `single_lookup_response`. An `Err` here should always mean /// the lookup is dropped. - fn handle_verified_response>( + fn handle_verified_response>( &self, seen_timestamp: Duration, cx: &mut SyncNetworkContext, process_type: BlockProcessType, verified_response: R::VerifiedResponseType, - lookup: &mut SingleBlockLookup, + lookup: &mut SingleBlockLookup, ) -> Result<(), LookupRequestError> { let id = lookup.id; let block_root = lookup.block_root(); @@ -389,7 +387,7 @@ impl BlockLookups { // If we have an outstanding parent request for this block, delay sending the response until // all parent blocks have been processed, otherwise we will fail validation with an // `UnknownParent`. - let delay_send = match L::lookup_type() { + let delay_send = match lookup.lookup_type { LookupType::Parent => false, LookupType::Current => self.has_pending_parent_request(lookup.block_root()), }; @@ -453,7 +451,7 @@ impl BlockLookups { /// Get a parent block lookup by its ID. This method additionally ensures the `req_counter` /// matches the current `req_counter` for the lookup. This any stale responses from requests /// that have been retried are ignored. - fn get_parent_lookup>( + fn get_parent_lookup>( &mut self, id: SingleLookupReqId, ) -> Option> { @@ -479,7 +477,7 @@ impl BlockLookups { } /// Process a response received from a parent lookup request. - pub fn parent_lookup_response>( + pub fn parent_lookup_response>( &mut self, id: SingleLookupReqId, peer_id: PeerId, @@ -523,7 +521,7 @@ impl BlockLookups { /// Consolidates error handling for `parent_lookup_response`. An `Err` here should always mean /// the lookup is dropped. - fn parent_lookup_response_inner>( + fn parent_lookup_response_inner>( &mut self, peer_id: PeerId, response: R::VerifiedResponseType, @@ -554,7 +552,7 @@ impl BlockLookups { } } - self.handle_verified_response::( + self.handle_verified_response::( seen_timestamp, cx, BlockProcessType::ParentLookup { @@ -633,7 +631,7 @@ impl BlockLookups { } /// An RPC error has occurred during a parent lookup. This function handles this case. - pub fn parent_lookup_failed>( + pub fn parent_lookup_failed>( &mut self, id: SingleLookupReqId, peer_id: &PeerId, @@ -669,7 +667,7 @@ impl BlockLookups { } /// An RPC error has occurred during a single lookup. This function handles this case.\ - pub fn single_block_lookup_failed>( + pub fn single_block_lookup_failed>( &mut self, id: SingleLookupReqId, peer_id: &PeerId, @@ -717,7 +715,7 @@ impl BlockLookups { /* Processing responses */ - pub fn single_block_component_processed>( + pub fn single_block_component_processed>( &mut self, target_id: Id, result: BlockProcessingResult, diff --git a/beacon_node/network/src/sync/block_lookups/parent_lookup.rs b/beacon_node/network/src/sync/block_lookups/parent_lookup.rs index b7a71860bff..11eb908953f 100644 --- a/beacon_node/network/src/sync/block_lookups/parent_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/parent_lookup.rs @@ -1,6 +1,6 @@ +use super::common::LookupType; use super::single_block_lookup::{LookupRequestError, SingleBlockLookup}; use super::{DownloadedBlock, PeerId}; -use crate::sync::block_lookups::common::Parent; use crate::sync::{manager::SLOT_IMPORT_TOLERANCE, network_context::SyncNetworkContext}; use beacon_chain::block_verification_types::AsBlock; use beacon_chain::block_verification_types::RpcBlock; @@ -24,7 +24,7 @@ pub(crate) struct ParentLookup { /// The blocks that have currently been downloaded. downloaded_blocks: Vec>, /// Request of the last parent. - pub current_parent_request: SingleBlockLookup, + pub current_parent_request: SingleBlockLookup, } #[derive(Debug, PartialEq, Eq)] @@ -55,6 +55,7 @@ impl ParentLookup { &[peer_id], da_checker, cx.next_id(), + LookupType::Parent, ); Self { @@ -132,7 +133,7 @@ impl ParentLookup { Hash256, VecDeque>, Vec, - SingleBlockLookup, + SingleBlockLookup, ) { let ParentLookup { chain_hash, diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index 5bb663967d7..077af7c3d19 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -1,5 +1,6 @@ +use super::common::LookupType; use super::PeerId; -use crate::sync::block_lookups::common::{Lookup, RequestState}; +use crate::sync::block_lookups::common::RequestState; use crate::sync::block_lookups::Id; use crate::sync::network_context::SyncNetworkContext; use beacon_chain::block_verification_types::RpcBlock; @@ -14,7 +15,6 @@ use rand::seq::IteratorRandom; use slog::{debug, Logger}; use std::collections::HashSet; use std::fmt::Debug; -use std::marker::PhantomData; use std::sync::Arc; use store::Hash256; use strum::IntoStaticStr; @@ -33,27 +33,30 @@ pub enum LookupRequestError { BadState(String), } -pub struct SingleBlockLookup { +pub struct SingleBlockLookup { pub id: Id, - pub block_request_state: BlockRequestState, - pub blob_request_state: BlobRequestState, + pub lookup_type: LookupType, + pub block_request_state: BlockRequestState, + pub blob_request_state: BlobRequestState, pub da_checker: Arc>, /// Only necessary for requests triggered by an `UnknownBlockParent` or `UnknownBlockParent` /// because any blocks or blobs without parents won't hit the data availability cache. pub child_components: Option>, } -impl SingleBlockLookup { +impl SingleBlockLookup { pub fn new( requested_block_root: Hash256, child_components: Option>, peers: &[PeerId], da_checker: Arc>, id: Id, + lookup_type: LookupType, ) -> Self { let is_deneb = da_checker.is_deneb(); Self { id, + lookup_type, block_request_state: BlockRequestState::new(requested_block_root, peers), blob_request_state: BlobRequestState::new(requested_block_root, peers, is_deneb), da_checker, @@ -103,11 +106,11 @@ impl SingleBlockLookup { if !block_already_downloaded { self.block_request_state - .build_request_and_send(self.id, cx)?; + .build_request_and_send(self.id, self.lookup_type, cx)?; } if !blobs_already_downloaded { self.blob_request_state - .build_request_and_send(self.id, cx)?; + .build_request_and_send(self.id, self.lookup_type, cx)?; } Ok(()) } @@ -144,7 +147,7 @@ impl SingleBlockLookup { /// Accepts a verified response, and adds it to the child components if required. This method /// returns a `CachedChild` which provides a completed block + blob response if all components have been /// received, or information about whether the child is required and if it has been downloaded. - pub fn add_response>( + pub fn add_response>( &mut self, verified_response: R::VerifiedResponseType, ) -> CachedChild { @@ -301,7 +304,7 @@ impl SingleBlockLookup { } /// The state of the blob request component of a `SingleBlockLookup`. -pub struct BlobRequestState { +pub struct BlobRequestState { /// The latest picture of which blobs still need to be requested. This includes information /// from both block/blobs downloaded in the network layer and any blocks/blobs that exist in /// the data availability checker. @@ -310,10 +313,9 @@ pub struct BlobRequestState { /// Where we store blobs until we receive the stream terminator. pub blob_download_queue: FixedBlobSidecarList, pub state: SingleLookupRequestState, - _phantom: PhantomData, } -impl BlobRequestState { +impl BlobRequestState { pub fn new(block_root: Hash256, peer_source: &[PeerId], is_deneb: bool) -> Self { let default_ids = MissingBlobs::new_without_block(block_root, is_deneb); Self { @@ -321,24 +323,21 @@ impl BlobRequestState { requested_ids: default_ids, blob_download_queue: <_>::default(), state: SingleLookupRequestState::new(peer_source), - _phantom: PhantomData, } } } /// The state of the block request component of a `SingleBlockLookup`. -pub struct BlockRequestState { +pub struct BlockRequestState { pub requested_block_root: Hash256, pub state: SingleLookupRequestState, - _phantom: PhantomData, } -impl BlockRequestState { +impl BlockRequestState { pub fn new(block_root: Hash256, peers: &[PeerId]) -> Self { Self { requested_block_root: block_root, state: SingleLookupRequestState::new(peers), - _phantom: PhantomData, } } } @@ -525,7 +524,7 @@ impl SingleLookupRequestState { } } -impl slog::Value for SingleBlockLookup { +impl slog::Value for SingleBlockLookup { fn serialize( &self, _record: &slog::Record, @@ -533,7 +532,7 @@ impl slog::Value for SingleBlockLookup { serializer: &mut dyn slog::Serializer, ) -> slog::Result { serializer.emit_str("request", key)?; - serializer.emit_arguments("lookup_type", &format_args!("{:?}", L::lookup_type()))?; + serializer.emit_arguments("lookup_type", &format_args!("{:?}", self.lookup_type))?; serializer.emit_arguments("hash", &format_args!("{}", self.block_root()))?; serializer.emit_arguments( "blob_ids", @@ -587,138 +586,3 @@ impl std::fmt::Display for State { } } } - -#[cfg(test)] -mod tests { - use super::*; - use crate::sync::block_lookups::common::LookupType; - use beacon_chain::builder::Witness; - use beacon_chain::eth1_chain::CachingEth1Backend; - use sloggers::null::NullLoggerBuilder; - use sloggers::Build; - use slot_clock::{SlotClock, TestingSlotClock}; - use std::time::Duration; - use store::{HotColdDB, MemoryStore, StoreConfig}; - use types::{ - test_utils::{SeedableRng, TestRandom, XorShiftRng}, - ChainSpec, MinimalEthSpec as E, SignedBeaconBlock, Slot, - }; - - fn rand_block() -> SignedBeaconBlock { - let mut rng = XorShiftRng::from_seed([42; 16]); - SignedBeaconBlock::from_block( - types::BeaconBlock::Base(types::BeaconBlockBase { - ..<_>::random_for_test(&mut rng) - }), - types::Signature::random_for_test(&mut rng), - ) - } - type T = Witness, E, MemoryStore, MemoryStore>; - - struct TestLookup1; - - impl Lookup for TestLookup1 { - const MAX_ATTEMPTS: u8 = 3; - - fn lookup_type() -> LookupType { - panic!() - } - } - - struct TestLookup2; - - impl Lookup for TestLookup2 { - const MAX_ATTEMPTS: u8 = 4; - - fn lookup_type() -> LookupType { - panic!() - } - } - - #[test] - fn test_happy_path() { - let peer_id = PeerId::random(); - let block = rand_block(); - let spec = E::default_spec(); - let slot_clock = TestingSlotClock::new( - Slot::new(0), - Duration::from_secs(0), - Duration::from_secs(spec.seconds_per_slot), - ); - let log = NullLoggerBuilder.build().expect("logger should build"); - let store = - HotColdDB::open_ephemeral(StoreConfig::default(), ChainSpec::minimal(), log.clone()) - .expect("store"); - let da_checker = Arc::new( - DataAvailabilityChecker::new(slot_clock, None, store.into(), &log, spec.clone()) - .expect("data availability checker"), - ); - let mut sl = SingleBlockLookup::::new( - block.canonical_root(), - None, - &[peer_id], - da_checker, - 1, - ); - as RequestState>::build_request( - &mut sl.block_request_state, - ) - .unwrap(); - sl.block_request_state.state.state = State::Downloading { peer_id }; - } - - #[test] - fn test_block_lookup_failures() { - let peer_id = PeerId::random(); - let block = rand_block(); - let spec = E::default_spec(); - let slot_clock = TestingSlotClock::new( - Slot::new(0), - Duration::from_secs(0), - Duration::from_secs(spec.seconds_per_slot), - ); - let log = NullLoggerBuilder.build().expect("logger should build"); - let store = - HotColdDB::open_ephemeral(StoreConfig::default(), ChainSpec::minimal(), log.clone()) - .expect("store"); - - let da_checker = Arc::new( - DataAvailabilityChecker::new(slot_clock, None, store.into(), &log, spec.clone()) - .expect("data availability checker"), - ); - - let mut sl = SingleBlockLookup::::new( - block.canonical_root(), - None, - &[peer_id], - da_checker, - 1, - ); - for _ in 1..TestLookup2::MAX_ATTEMPTS { - as RequestState>::build_request( - &mut sl.block_request_state, - ) - .unwrap(); - sl.block_request_state.state.on_download_failure(); - } - - // Now we receive the block and send it for processing - as RequestState>::build_request( - &mut sl.block_request_state, - ) - .unwrap(); - sl.block_request_state.state.state = State::Downloading { peer_id }; - - // One processing failure maxes the available attempts - sl.block_request_state.state.on_processing_failure(); - assert_eq!( - as RequestState>::build_request( - &mut sl.block_request_state, - ) - .unwrap_err(), - LookupRequestError::TooManyAttempts { - cannot_process: false - } - ) - } -} diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 23bd1010bfe..9c17c6a1512 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -42,7 +42,6 @@ use super::range_sync::{RangeSync, RangeSyncType, EPOCHS_PER_BATCH}; use crate::network_beacon_processor::{ChainSegmentProcessId, NetworkBeaconProcessor}; use crate::service::NetworkMessage; use crate::status::ToStatusMessage; -use crate::sync::block_lookups::common::{Current, Parent}; use crate::sync::block_lookups::{BlobRequestState, BlockRequestState}; use crate::sync::block_sidecar_coupling::BlocksAndBlobsRequestInfo; use beacon_chain::block_verification_types::AsBlock; @@ -621,14 +620,14 @@ impl SyncManager { } => match process_type { BlockProcessType::SingleBlock { id } => self .block_lookups - .single_block_component_processed::>( + .single_block_component_processed::( id, result, &mut self.network, ), BlockProcessType::SingleBlob { id } => self .block_lookups - .single_block_component_processed::>( + .single_block_component_processed::>( id, result, &mut self.network, @@ -834,7 +833,7 @@ impl SyncManager { Ok((block, seen_timestamp)) => match id.lookup_type { LookupType::Current => self .block_lookups - .single_lookup_response::>( + .single_lookup_response::( id, peer_id, block, @@ -843,7 +842,7 @@ impl SyncManager { ), LookupType::Parent => self .block_lookups - .parent_lookup_response::>( + .parent_lookup_response::( id, peer_id, block, @@ -854,7 +853,7 @@ impl SyncManager { Err(error) => match id.lookup_type { LookupType::Current => self .block_lookups - .single_block_lookup_failed::>( + .single_block_lookup_failed::( id, &peer_id, &mut self.network, @@ -862,7 +861,7 @@ impl SyncManager { ), LookupType::Parent => self .block_lookups - .parent_lookup_failed::>( + .parent_lookup_failed::( id, &peer_id, &mut self.network, @@ -909,7 +908,7 @@ impl SyncManager { Ok((blobs, seen_timestamp)) => match id.lookup_type { LookupType::Current => self .block_lookups - .single_lookup_response::>( + .single_lookup_response::>( id, peer_id, blobs, @@ -918,7 +917,7 @@ impl SyncManager { ), LookupType::Parent => self .block_lookups - .parent_lookup_response::>( + .parent_lookup_response::>( id, peer_id, blobs, @@ -930,7 +929,7 @@ impl SyncManager { Err(error) => match id.lookup_type { LookupType::Current => self .block_lookups - .single_block_lookup_failed::>( + .single_block_lookup_failed::>( id, &peer_id, &mut self.network, @@ -938,7 +937,7 @@ impl SyncManager { ), LookupType::Parent => self .block_lookups - .parent_lookup_failed::>( + .parent_lookup_failed::>( id, &peer_id, &mut self.network, From 4a48d7b546c21b668b12050947b3ea33cf973fd5 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Wed, 24 Apr 2024 01:02:48 -0500 Subject: [PATCH 28/37] Encode Execution Engine Client Version In Graffiti (#5290) * Add `engine_clientVersionV1` structs * Implement `engine_clientVersionV1` * Update to latest spec changes * Implement GraffitiCalculator Service * Added Unit Tests for GraffitiCalculator * Address Mac's Comments * Remove need to use clap in beacon chain * Merge remote-tracking branch 'upstream/unstable' into el_client_version_graffiti * Merge branch 'unstable' into el_client_version_graffiti # Conflicts: # beacon_node/beacon_chain/Cargo.toml --- Cargo.lock | 2 + beacon_node/beacon_chain/Cargo.toml | 1 + beacon_node/beacon_chain/src/beacon_chain.rs | 17 +- beacon_node/beacon_chain/src/builder.rs | 24 +- .../beacon_chain/src/graffiti_calculator.rs | 377 ++++++++++++++++++ beacon_node/beacon_chain/src/lib.rs | 1 + beacon_node/client/src/builder.rs | 9 +- beacon_node/client/src/config.rs | 8 +- beacon_node/execution_layer/Cargo.toml | 1 + beacon_node/execution_layer/src/engine_api.rs | 145 ++++++- .../execution_layer/src/engine_api/http.rs | 111 +++++- .../src/engine_api/json_structures.rs | 33 ++ beacon_node/execution_layer/src/engines.rs | 57 ++- beacon_node/execution_layer/src/lib.rs | 33 +- .../src/test_utils/handle_rpc.rs | 5 +- .../execution_layer/src/test_utils/mod.rs | 12 + beacon_node/src/config.rs | 27 +- common/lighthouse_version/src/lib.rs | 18 + consensus/types/src/graffiti.rs | 6 + lighthouse/tests/beacon_node.rs | 41 +- 20 files changed, 847 insertions(+), 81 deletions(-) create mode 100644 beacon_node/beacon_chain/src/graffiti_calculator.rs diff --git a/Cargo.lock b/Cargo.lock index 335491adc1c..cc38877fe76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -774,6 +774,7 @@ dependencies = [ "kzg", "lazy_static", "lighthouse_metrics", + "lighthouse_version", "logging", "lru", "maplit", @@ -2845,6 +2846,7 @@ dependencies = [ "kzg", "lazy_static", "lighthouse_metrics", + "lighthouse_version", "lru", "parking_lot 0.12.1", "pretty_reqwest_error", diff --git a/beacon_node/beacon_chain/Cargo.toml b/beacon_node/beacon_chain/Cargo.toml index 9c7c7febc5f..32a1056c10e 100644 --- a/beacon_node/beacon_chain/Cargo.toml +++ b/beacon_node/beacon_chain/Cargo.toml @@ -39,6 +39,7 @@ itertools = { workspace = true } kzg = { workspace = true } lazy_static = { workspace = true } lighthouse_metrics = { workspace = true } +lighthouse_version = { workspace = true } logging = { workspace = true } lru = { workspace = true } merkle_proof = { workspace = true } diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index c65dfdb449f..dd1fe59b922 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -30,6 +30,7 @@ use crate::eth1_finalization_cache::{Eth1FinalizationCache, Eth1FinalizationData use crate::events::ServerSentEventHandler; use crate::execution_payload::{get_execution_payload, NotifyExecutionLayer, PreparePayloadHandle}; use crate::fork_choice_signal::{ForkChoiceSignalRx, ForkChoiceSignalTx, ForkChoiceWaitResult}; +use crate::graffiti_calculator::GraffitiCalculator; use crate::head_tracker::{HeadTracker, HeadTrackerReader, SszHeadTracker}; use crate::historical_blocks::HistoricalBlockError; use crate::light_client_finality_update_verification::{ @@ -474,7 +475,7 @@ pub struct BeaconChain { /// Logging to CLI, etc. pub(crate) log: Logger, /// Arbitrary bytes included in the blocks. - pub(crate) graffiti: Graffiti, + pub(crate) graffiti_calculator: GraffitiCalculator, /// Optional slasher. pub slasher: Option>>, /// Provides monitoring of a set of explicitly defined validators. @@ -4654,6 +4655,10 @@ impl BeaconChain { // // Perform the state advance and block-packing functions. let chain = self.clone(); + let graffiti = self + .graffiti_calculator + .get_graffiti(validator_graffiti) + .await; let mut partial_beacon_block = self .task_executor .spawn_blocking_handle( @@ -4663,7 +4668,7 @@ impl BeaconChain { state_root_opt, produce_at_slot, randao_reveal, - validator_graffiti, + graffiti, builder_boost_factor, block_production_version, ) @@ -4761,7 +4766,7 @@ impl BeaconChain { state_root_opt: Option, produce_at_slot: Slot, randao_reveal: Signature, - validator_graffiti: Option, + graffiti: Graffiti, builder_boost_factor: Option, block_production_version: BlockProductionVersion, ) -> Result, BlockProductionError> { @@ -4867,12 +4872,6 @@ impl BeaconChain { } drop(unagg_import_timer); - // Override the beacon node's graffiti with graffiti from the validator, if present. - let graffiti = match validator_graffiti { - Some(graffiti) => graffiti, - None => self.graffiti, - }; - let attestation_packing_timer = metrics::start_timer(&metrics::BLOCK_PRODUCTION_ATTESTATION_TIMES); diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index be6b1f9b2bf..376bc16c035 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -7,6 +7,7 @@ use crate::eth1_chain::{CachingEth1Backend, SszEth1}; use crate::eth1_finalization_cache::Eth1FinalizationCache; use crate::fork_choice_signal::ForkChoiceSignalTx; use crate::fork_revert::{reset_fork_choice_to_finalization, revert_to_fork_boundary}; +use crate::graffiti_calculator::{GraffitiCalculator, GraffitiOrigin}; use crate::head_tracker::HeadTracker; use crate::light_client_server_cache::LightClientServerCache; use crate::migrate::{BackgroundMigrator, MigratorConfig}; @@ -38,8 +39,8 @@ use std::time::Duration; use store::{Error as StoreError, HotColdDB, ItemStore, KeyValueStoreOp}; use task_executor::{ShutdownReason, TaskExecutor}; use types::{ - BeaconBlock, BeaconState, BlobSidecarList, ChainSpec, Checkpoint, Epoch, EthSpec, Graffiti, - Hash256, Signature, SignedBeaconBlock, Slot, + BeaconBlock, BeaconState, BlobSidecarList, ChainSpec, Checkpoint, Epoch, EthSpec, Hash256, + Signature, SignedBeaconBlock, Slot, }; /// An empty struct used to "witness" all the `BeaconChainTypes` traits. It has no user-facing @@ -95,7 +96,7 @@ pub struct BeaconChainBuilder { spec: ChainSpec, chain_config: ChainConfig, log: Option, - graffiti: Graffiti, + beacon_graffiti: GraffitiOrigin, slasher: Option>>, // Pending I/O batch that is constructed during building and should be executed atomically // alongside `PersistedBeaconChain` storage when `BeaconChainBuilder::build` is called. @@ -138,7 +139,7 @@ where spec: E::default_spec(), chain_config: ChainConfig::default(), log: None, - graffiti: Graffiti::default(), + beacon_graffiti: GraffitiOrigin::default(), slasher: None, pending_io_batch: vec![], kzg: None, @@ -655,9 +656,9 @@ where self } - /// Sets the `graffiti` field. - pub fn graffiti(mut self, graffiti: Graffiti) -> Self { - self.graffiti = graffiti; + /// Sets the `beacon_graffiti` field. + pub fn beacon_graffiti(mut self, beacon_graffiti: GraffitiOrigin) -> Self { + self.beacon_graffiti = beacon_graffiti; self } @@ -924,7 +925,7 @@ where observed_attester_slashings: <_>::default(), observed_bls_to_execution_changes: <_>::default(), eth1_chain: self.eth1_chain, - execution_layer: self.execution_layer, + execution_layer: self.execution_layer.clone(), genesis_validators_root, genesis_time, canonical_head, @@ -953,7 +954,12 @@ where .shutdown_sender .ok_or("Cannot build without a shutdown sender.")?, log: log.clone(), - graffiti: self.graffiti, + graffiti_calculator: GraffitiCalculator::new( + self.beacon_graffiti, + self.execution_layer, + slot_clock.slot_duration() * E::slots_per_epoch() as u32, + log.clone(), + ), slasher: self.slasher.clone(), validator_monitor: RwLock::new(validator_monitor), genesis_backfill_slot, diff --git a/beacon_node/beacon_chain/src/graffiti_calculator.rs b/beacon_node/beacon_chain/src/graffiti_calculator.rs new file mode 100644 index 00000000000..599c99dc2da --- /dev/null +++ b/beacon_node/beacon_chain/src/graffiti_calculator.rs @@ -0,0 +1,377 @@ +use crate::BeaconChain; +use crate::BeaconChainTypes; +use execution_layer::{http::ENGINE_GET_CLIENT_VERSION_V1, CommitPrefix, ExecutionLayer}; +use serde::{Deserialize, Serialize}; +use slog::{crit, debug, error, warn, Logger}; +use slot_clock::SlotClock; +use std::{fmt::Debug, time::Duration}; +use task_executor::TaskExecutor; +use types::{EthSpec, Graffiti, GRAFFITI_BYTES_LEN}; + +const ENGINE_VERSION_AGE_LIMIT_EPOCH_MULTIPLE: u32 = 6; // 6 epochs +const ENGINE_VERSION_CACHE_REFRESH_EPOCH_MULTIPLE: u32 = 2; // 2 epochs +const ENGINE_VERSION_CACHE_PRELOAD_STARTUP_DELAY: Duration = Duration::from_secs(60); + +/// Represents the source and content of graffiti for block production, excluding +/// inputs from the validator client and execution engine. Graffiti is categorized +/// as either user-specified or calculated to facilitate decisions on graffiti +/// selection. +#[derive(Clone, Copy, Serialize, Deserialize)] +pub enum GraffitiOrigin { + UserSpecified(Graffiti), + Calculated(Graffiti), +} + +impl GraffitiOrigin { + pub fn graffiti(&self) -> Graffiti { + match self { + GraffitiOrigin::UserSpecified(graffiti) => *graffiti, + GraffitiOrigin::Calculated(graffiti) => *graffiti, + } + } +} + +impl Default for GraffitiOrigin { + fn default() -> Self { + let version_bytes = lighthouse_version::VERSION.as_bytes(); + let trimmed_len = std::cmp::min(version_bytes.len(), GRAFFITI_BYTES_LEN); + let mut bytes = [0u8; GRAFFITI_BYTES_LEN]; + bytes[..trimmed_len].copy_from_slice(&version_bytes[..trimmed_len]); + Self::Calculated(Graffiti::from(bytes)) + } +} + +impl Debug for GraffitiOrigin { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + self.graffiti().fmt(f) + } +} + +pub struct GraffitiCalculator { + pub beacon_graffiti: GraffitiOrigin, + execution_layer: Option>, + pub epoch_duration: Duration, + log: Logger, +} + +impl GraffitiCalculator { + pub fn new( + beacon_graffiti: GraffitiOrigin, + execution_layer: Option>, + epoch_duration: Duration, + log: Logger, + ) -> Self { + Self { + beacon_graffiti, + execution_layer, + epoch_duration, + log, + } + } + + /// Returns the appropriate graffiti to use for block production, prioritizing + /// sources in the following order: + /// 1. Graffiti specified by the validator client. + /// 2. Graffiti specified by the user via beacon node CLI options. + /// 3. The EL & CL client version string, applicable when the EL supports version specification. + /// 4. The default lighthouse version string, used if the EL lacks version specification support. + pub async fn get_graffiti(&self, validator_graffiti: Option) -> Graffiti { + if let Some(graffiti) = validator_graffiti { + return graffiti; + } + + match self.beacon_graffiti { + GraffitiOrigin::UserSpecified(graffiti) => graffiti, + GraffitiOrigin::Calculated(default_graffiti) => { + let Some(execution_layer) = self.execution_layer.as_ref() else { + // Return default graffiti if there is no execution layer. This + // shouldn't occur if we're actually producing blocks. + crit!(self.log, "No execution layer available for graffiti calculation during block production!"); + return default_graffiti; + }; + + // The engine version cache refresh service ensures this will almost always retrieve this data from the + // cache instead of making a request to the execution engine. A cache miss would only occur if lighthouse + // has recently started or the EL recently went offline. + let engine_versions = match execution_layer + .get_engine_version(Some( + self.epoch_duration * ENGINE_VERSION_AGE_LIMIT_EPOCH_MULTIPLE, + )) + .await + { + Ok(engine_versions) => engine_versions, + Err(el_error) => { + warn!(self.log, "Failed to determine execution engine version for graffiti"; "error" => ?el_error); + return default_graffiti; + } + }; + + let Some(engine_version) = engine_versions.first() else { + // Got an empty array which indicates the EL doesn't support the method + debug!( + self.log, + "Using default lighthouse graffiti: EL does not support {} method", + ENGINE_GET_CLIENT_VERSION_V1; + ); + return default_graffiti; + }; + if engine_versions.len() != 1 { + // More than one version implies lighthouse is connected to + // an EL multiplexer. We don't support modifying the graffiti + // with these configurations. + warn!( + self.log, + "Execution Engine multiplexer detected, using default graffiti" + ); + return default_graffiti; + } + + let lighthouse_commit_prefix = CommitPrefix::try_from(lighthouse_version::COMMIT_PREFIX.to_string()) + .unwrap_or_else(|error_message| { + // This really shouldn't happen but we want to definitly log if it does + crit!(self.log, "Failed to parse lighthouse commit prefix"; "error" => error_message); + CommitPrefix("00000000".to_string()) + }); + + engine_version.calculate_graffiti(lighthouse_commit_prefix) + } + } + } +} + +pub fn start_engine_version_cache_refresh_service( + chain: &BeaconChain, + executor: TaskExecutor, +) { + let Some(el_ref) = chain.execution_layer.as_ref() else { + debug!( + chain.log, + "No execution layer configured, not starting engine version cache refresh service" + ); + return; + }; + if matches!( + chain.graffiti_calculator.beacon_graffiti, + GraffitiOrigin::UserSpecified(_) + ) { + debug!( + chain.log, + "Graffiti is user-specified, not starting engine version cache refresh service" + ); + return; + } + + let execution_layer = el_ref.clone(); + let log = chain.log.clone(); + let slot_clock = chain.slot_clock.clone(); + let epoch_duration = chain.graffiti_calculator.epoch_duration; + executor.spawn( + async move { + engine_version_cache_refresh_service::( + execution_layer, + slot_clock, + epoch_duration, + log, + ) + .await + }, + "engine_version_cache_refresh_service", + ); +} + +async fn engine_version_cache_refresh_service( + execution_layer: ExecutionLayer, + slot_clock: T::SlotClock, + epoch_duration: Duration, + log: Logger, +) { + // Preload the engine version cache after a brief delay to allow for EL initialization. + // This initial priming ensures cache readiness before the service's regular update cycle begins. + tokio::time::sleep(ENGINE_VERSION_CACHE_PRELOAD_STARTUP_DELAY).await; + if let Err(e) = execution_layer.get_engine_version(None).await { + debug!(log, "Failed to preload engine version cache"; "error" => format!("{:?}", e)); + } + + // this service should run 3/8 of the way through the epoch + let epoch_delay = (epoch_duration * 3) / 8; + // the duration of 1 epoch less than the total duration between firing of this service + let partial_firing_delay = + epoch_duration * ENGINE_VERSION_CACHE_REFRESH_EPOCH_MULTIPLE.saturating_sub(1); + loop { + match slot_clock.duration_to_next_epoch(T::EthSpec::slots_per_epoch()) { + Some(duration_to_next_epoch) => { + let firing_delay = partial_firing_delay + duration_to_next_epoch + epoch_delay; + tokio::time::sleep(firing_delay).await; + + debug!( + log, + "Engine version cache refresh service firing"; + ); + + match execution_layer.get_engine_version(None).await { + Err(e) => warn!(log, "Failed to populate engine version cache"; "error" => ?e), + Ok(versions) => { + if versions.is_empty() { + // Empty array indicates the EL doesn't support the method + debug!( + log, + "EL does not support {} method. Sleeping twice as long before retry", + ENGINE_GET_CLIENT_VERSION_V1 + ); + tokio::time::sleep( + epoch_duration * ENGINE_VERSION_CACHE_REFRESH_EPOCH_MULTIPLE, + ) + .await; + } + } + } + } + None => { + error!(log, "Failed to read slot clock"); + // If we can't read the slot clock, just wait another slot. + tokio::time::sleep(slot_clock.slot_duration()).await; + } + }; + } +} + +#[cfg(test)] +mod tests { + use crate::test_utils::{test_spec, BeaconChainHarness, EphemeralHarnessType}; + use crate::ChainConfig; + use execution_layer::test_utils::{DEFAULT_CLIENT_VERSION, DEFAULT_ENGINE_CAPABILITIES}; + use execution_layer::EngineCapabilities; + use lazy_static::lazy_static; + use slog::info; + use std::time::Duration; + use types::{ChainSpec, Graffiti, Keypair, MinimalEthSpec, GRAFFITI_BYTES_LEN}; + + const VALIDATOR_COUNT: usize = 48; + lazy_static! { + /// A cached set of keys. + static ref KEYPAIRS: Vec = types::test_utils::generate_deterministic_keypairs(VALIDATOR_COUNT); + } + + fn get_harness( + validator_count: usize, + spec: ChainSpec, + chain_config: Option, + ) -> BeaconChainHarness> { + let harness = BeaconChainHarness::builder(MinimalEthSpec) + .spec(spec) + .chain_config(chain_config.unwrap_or_default()) + .keypairs(KEYPAIRS[0..validator_count].to_vec()) + .logger(logging::test_logger()) + .fresh_ephemeral_store() + .mock_execution_layer() + .build(); + + harness.advance_slot(); + + harness + } + + #[tokio::test] + async fn check_graffiti_without_el_version_support() { + let spec = test_spec::(); + let harness = get_harness(VALIDATOR_COUNT, spec, None); + // modify execution engine so it doesn't support engine_getClientVersionV1 method + let mock_execution_layer = harness.mock_execution_layer.as_ref().unwrap(); + mock_execution_layer + .server + .set_engine_capabilities(EngineCapabilities { + get_client_version_v1: false, + ..DEFAULT_ENGINE_CAPABILITIES + }); + // refresh capabilities cache + harness + .chain + .execution_layer + .as_ref() + .unwrap() + .get_engine_capabilities(Some(Duration::ZERO)) + .await + .unwrap(); + + let version_bytes = std::cmp::min( + lighthouse_version::VERSION.as_bytes().len(), + GRAFFITI_BYTES_LEN, + ); + // grab the slice of the graffiti that corresponds to the lighthouse version + let graffiti_slice = + &harness.chain.graffiti_calculator.get_graffiti(None).await.0[..version_bytes]; + + // convert graffiti bytes slice to ascii for easy debugging if this test should fail + let graffiti_str = + std::str::from_utf8(graffiti_slice).expect("bytes should convert nicely to ascii"); + + info!(harness.chain.log, "results"; "lighthouse_version" => lighthouse_version::VERSION, "graffiti_str" => graffiti_str); + println!("lighthouse_version: '{}'", lighthouse_version::VERSION); + println!("graffiti_str: '{}'", graffiti_str); + + assert!(lighthouse_version::VERSION.starts_with(graffiti_str)); + } + + #[tokio::test] + async fn check_graffiti_with_el_version_support() { + let spec = test_spec::(); + let harness = get_harness(VALIDATOR_COUNT, spec, None); + + let found_graffiti_bytes = harness.chain.graffiti_calculator.get_graffiti(None).await.0; + + let mock_commit = DEFAULT_CLIENT_VERSION.commit.clone(); + let expected_graffiti_string = format!( + "{}{}{}{}", + DEFAULT_CLIENT_VERSION.code, + mock_commit + .strip_prefix("0x") + .unwrap_or(&mock_commit) + .get(0..4) + .expect("should get first 2 bytes in hex"), + "LH", + lighthouse_version::COMMIT_PREFIX + .get(0..4) + .expect("should get first 2 bytes in hex") + ); + + let expected_graffiti_prefix_bytes = expected_graffiti_string.as_bytes(); + let expected_graffiti_prefix_len = + std::cmp::min(expected_graffiti_prefix_bytes.len(), GRAFFITI_BYTES_LEN); + + let found_graffiti_string = + std::str::from_utf8(&found_graffiti_bytes[..expected_graffiti_prefix_len]) + .expect("bytes should convert nicely to ascii"); + + info!(harness.chain.log, "results"; "expected_graffiti_string" => &expected_graffiti_string, "found_graffiti_string" => &found_graffiti_string); + println!("expected_graffiti_string: '{}'", expected_graffiti_string); + println!("found_graffiti_string: '{}'", found_graffiti_string); + + assert_eq!(expected_graffiti_string, found_graffiti_string); + + let mut expected_graffiti_bytes = [0u8; GRAFFITI_BYTES_LEN]; + expected_graffiti_bytes[..expected_graffiti_prefix_len] + .copy_from_slice(expected_graffiti_string.as_bytes()); + assert_eq!(found_graffiti_bytes, expected_graffiti_bytes); + } + + #[tokio::test] + async fn check_graffiti_with_validator_specified_value() { + let spec = test_spec::(); + let harness = get_harness(VALIDATOR_COUNT, spec, None); + + let graffiti_str = "nice graffiti bro"; + let mut graffiti_bytes = [0u8; GRAFFITI_BYTES_LEN]; + graffiti_bytes[..graffiti_str.as_bytes().len()].copy_from_slice(graffiti_str.as_bytes()); + + let found_graffiti = harness + .chain + .graffiti_calculator + .get_graffiti(Some(Graffiti::from(graffiti_bytes))) + .await; + + assert_eq!( + found_graffiti.to_string(), + "0x6e6963652067726166666974692062726f000000000000000000000000000000" + ); + } +} diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index c1df9ede87a..5ee7803436c 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -28,6 +28,7 @@ pub mod events; pub mod execution_payload; pub mod fork_choice_signal; pub mod fork_revert; +pub mod graffiti_calculator; mod head_tracker; pub mod historical_blocks; pub mod kzg_utils; diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index e7f201b8521..2af4e74c224 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -6,6 +6,7 @@ use crate::notifier::spawn_notifier; use crate::Client; use beacon_chain::attestation_simulator::start_attestation_simulator_service; use beacon_chain::data_availability_checker::start_availability_cache_maintenance_service; +use beacon_chain::graffiti_calculator::start_engine_version_cache_refresh_service; use beacon_chain::otb_verification_service::start_otb_verification_service; use beacon_chain::proposer_prep_service::start_proposer_prep_service; use beacon_chain::schema_change::migrate_schema; @@ -164,7 +165,7 @@ where let runtime_context = self.runtime_context.clone(); let eth_spec_instance = self.eth_spec_instance.clone(); let chain_config = config.chain.clone(); - let graffiti = config.graffiti; + let beacon_graffiti = config.beacon_graffiti; let store = store.ok_or("beacon_chain_start_method requires a store")?; let runtime_context = @@ -203,7 +204,7 @@ where MigratorConfig::default().epochs_per_migration(chain_config.epochs_per_migration), ) .chain_config(chain_config) - .graffiti(graffiti) + .beacon_graffiti(beacon_graffiti) .event_handler(event_handler) .execution_layer(execution_layer) .validator_monitor_config(config.validator_monitor.clone()); @@ -967,6 +968,10 @@ where runtime_context.executor.clone(), beacon_chain.clone(), ); + start_engine_version_cache_refresh_service( + beacon_chain.as_ref(), + runtime_context.executor.clone(), + ); start_attestation_simulator_service( beacon_chain.task_executor.clone(), beacon_chain.clone(), diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index a441e2c186c..16000374b22 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -1,3 +1,4 @@ +use beacon_chain::graffiti_calculator::GraffitiOrigin; use beacon_chain::validator_monitor::ValidatorMonitorConfig; use beacon_chain::TrustedSetup; use beacon_processor::BeaconProcessorConfig; @@ -9,7 +10,6 @@ use serde::{Deserialize, Serialize}; use std::fs; use std::path::PathBuf; use std::time::Duration; -use types::Graffiti; /// Default directory name for the freezer database under the top-level data dir. const DEFAULT_FREEZER_DB_DIR: &str = "freezer_db"; @@ -63,8 +63,8 @@ pub struct Config { /// This is the method used for the 2019 client interop in Canada. pub dummy_eth1_backend: bool, pub sync_eth1_chain: bool, - /// Graffiti to be inserted everytime we create a block. - pub graffiti: Graffiti, + /// Graffiti to be inserted everytime we create a block if the validator doesn't specify. + pub beacon_graffiti: GraffitiOrigin, pub validator_monitor: ValidatorMonitorConfig, #[serde(skip)] /// The `genesis` field is not serialized or deserialized by `serde` to ensure it is defined @@ -104,7 +104,7 @@ impl Default for Config { eth1: <_>::default(), execution_layer: None, trusted_setup: None, - graffiti: Graffiti::default(), + beacon_graffiti: GraffitiOrigin::default(), http_api: <_>::default(), http_metrics: <_>::default(), monitoring_api: None, diff --git a/beacon_node/execution_layer/Cargo.toml b/beacon_node/execution_layer/Cargo.toml index ace8e24a8e4..28cd16e4ef9 100644 --- a/beacon_node/execution_layer/Cargo.toml +++ b/beacon_node/execution_layer/Cargo.toml @@ -52,3 +52,4 @@ arc-swap = "1.6.0" eth2_network_config = { workspace = true } alloy-rlp = "0.3" alloy-consensus = { git = "https://github.com/alloy-rs/alloy.git", rev = "974d488bab5e21e9f17452a39a4bfa56677367b2" } +lighthouse_version = { workspace = true } diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index a91f5d6a442..d422994c595 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -1,9 +1,9 @@ use crate::engines::ForkchoiceState; use crate::http::{ ENGINE_FORKCHOICE_UPDATED_V1, ENGINE_FORKCHOICE_UPDATED_V2, ENGINE_FORKCHOICE_UPDATED_V3, - ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1, ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1, - ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2, ENGINE_GET_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V1, - ENGINE_NEW_PAYLOAD_V2, ENGINE_NEW_PAYLOAD_V3, + ENGINE_GET_CLIENT_VERSION_V1, ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1, + ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1, ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2, + ENGINE_GET_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V1, ENGINE_NEW_PAYLOAD_V2, ENGINE_NEW_PAYLOAD_V3, }; use eth2::types::{ BlobsBundle, SsePayloadAttributes, SsePayloadAttributesV1, SsePayloadAttributesV2, @@ -24,11 +24,11 @@ pub use types::{ ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, Uint256, VariableList, Withdrawal, Withdrawals, }; - use types::{ ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadElectra, ExecutionPayloadMerge, KzgProofs, }; +use types::{Graffiti, GRAFFITI_BYTES_LEN}; pub mod auth; pub mod http; @@ -61,13 +61,13 @@ pub enum Error { ParentHashEqualsBlockHash(ExecutionBlockHash), PayloadIdUnavailable, TransitionConfigurationMismatch, - PayloadConversionLogicFlaw, SszError(ssz_types::Error), DeserializeWithdrawals(ssz_types::Error), BuilderApi(builder_client::Error), IncorrectStateVariant, RequiredMethodUnsupported(&'static str), UnsupportedForkVariant(String), + InvalidClientVersion(String), RlpDecoderError(rlp::DecoderError), } @@ -652,6 +652,7 @@ pub struct EngineCapabilities { pub get_payload_v1: bool, pub get_payload_v2: bool, pub get_payload_v3: bool, + pub get_client_version_v1: bool, } impl EngineCapabilities { @@ -690,7 +691,141 @@ impl EngineCapabilities { if self.get_payload_v3 { response.push(ENGINE_GET_PAYLOAD_V3); } + if self.get_client_version_v1 { + response.push(ENGINE_GET_CLIENT_VERSION_V1); + } response } } + +#[derive(Clone, Debug, PartialEq)] +pub enum ClientCode { + Besu, + EtherumJS, + Erigon, + GoEthereum, + Grandine, + Lighthouse, + Lodestar, + Nethermind, + Nimbus, + Teku, + Prysm, + Reth, + Unknown(String), +} + +impl std::fmt::Display for ClientCode { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let s = match self { + ClientCode::Besu => "BU", + ClientCode::EtherumJS => "EJ", + ClientCode::Erigon => "EG", + ClientCode::GoEthereum => "GE", + ClientCode::Grandine => "GR", + ClientCode::Lighthouse => "LH", + ClientCode::Lodestar => "LS", + ClientCode::Nethermind => "NM", + ClientCode::Nimbus => "NB", + ClientCode::Teku => "TK", + ClientCode::Prysm => "PM", + ClientCode::Reth => "RH", + ClientCode::Unknown(code) => code, + }; + write!(f, "{}", s) + } +} + +impl TryFrom for ClientCode { + type Error = String; + + fn try_from(code: String) -> Result { + match code.as_str() { + "BU" => Ok(Self::Besu), + "EJ" => Ok(Self::EtherumJS), + "EG" => Ok(Self::Erigon), + "GE" => Ok(Self::GoEthereum), + "GR" => Ok(Self::Grandine), + "LH" => Ok(Self::Lighthouse), + "LS" => Ok(Self::Lodestar), + "NM" => Ok(Self::Nethermind), + "NB" => Ok(Self::Nimbus), + "TK" => Ok(Self::Teku), + "PM" => Ok(Self::Prysm), + "RH" => Ok(Self::Reth), + string => { + if string.len() == 2 { + Ok(Self::Unknown(code)) + } else { + Err(format!("Invalid client code: {}", code)) + } + } + } + } +} + +#[derive(Clone, Debug)] +pub struct CommitPrefix(pub String); + +impl TryFrom for CommitPrefix { + type Error = String; + + fn try_from(value: String) -> Result { + // Check if the input starts with '0x' and strip it if it does + let commit_prefix = value.strip_prefix("0x").unwrap_or(&value); + + // Ensure length is exactly 8 characters after '0x' removal + if commit_prefix.len() != 8 { + return Err( + "Input must be exactly 8 characters long (excluding any '0x' prefix)".to_string(), + ); + } + + // Ensure all characters are valid hex digits + if commit_prefix.chars().all(|c| c.is_ascii_hexdigit()) { + Ok(CommitPrefix(commit_prefix.to_lowercase())) + } else { + Err("Input must contain only hexadecimal characters".to_string()) + } + } +} + +impl std::fmt::Display for CommitPrefix { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.0) + } +} + +#[derive(Clone, Debug)] +pub struct ClientVersionV1 { + pub code: ClientCode, + pub name: String, + pub version: String, + pub commit: CommitPrefix, +} + +impl ClientVersionV1 { + pub fn calculate_graffiti(&self, lighthouse_commit_prefix: CommitPrefix) -> Graffiti { + let graffiti_string = format!( + "{}{}LH{}", + self.code, + self.commit + .0 + .get(..4) + .map_or_else(|| self.commit.0.as_str(), |s| s) + .to_lowercase(), + lighthouse_commit_prefix + .0 + .get(..4) + .unwrap_or("0000") + .to_lowercase(), + ); + let mut graffiti_bytes = [0u8; GRAFFITI_BYTES_LEN]; + let bytes_to_copy = std::cmp::min(graffiti_string.len(), GRAFFITI_BYTES_LEN); + graffiti_bytes[..bytes_to_copy] + .copy_from_slice(&graffiti_string.as_bytes()[..bytes_to_copy]); + + Graffiti::from(graffiti_bytes) + } +} diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index ebd6ebeba2a..405416e4f9d 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -3,6 +3,8 @@ use super::*; use crate::auth::Auth; use crate::json_structures::*; +use lazy_static::lazy_static; +use lighthouse_version::{COMMIT_PREFIX, VERSION}; use reqwest::header::CONTENT_TYPE; use sensitive_url::SensitiveUrl; use serde::de::DeserializeOwned; @@ -51,6 +53,9 @@ pub const ENGINE_GET_PAYLOAD_BODIES_TIMEOUT: Duration = Duration::from_secs(10); pub const ENGINE_EXCHANGE_CAPABILITIES: &str = "engine_exchangeCapabilities"; pub const ENGINE_EXCHANGE_CAPABILITIES_TIMEOUT: Duration = Duration::from_secs(1); +pub const ENGINE_GET_CLIENT_VERSION_V1: &str = "engine_getClientVersionV1"; +pub const ENGINE_GET_CLIENT_VERSION_TIMEOUT: Duration = Duration::from_secs(1); + /// This error is returned during a `chainId` call by Geth. pub const EIP155_ERROR_STR: &str = "chain not synced beyond EIP-155 replay-protection fork block"; /// This code is returned by all clients when a method is not supported @@ -69,8 +74,22 @@ pub static LIGHTHOUSE_CAPABILITIES: &[&str] = &[ ENGINE_FORKCHOICE_UPDATED_V3, ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1, ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1, + ENGINE_GET_CLIENT_VERSION_V1, ]; +lazy_static! { + /// We opt to initialize the JsonClientVersionV1 rather than the ClientVersionV1 + /// for two reasons: + /// 1. This saves the overhead of converting into Json for every engine call + /// 2. The Json version lacks error checking so we can avoid calling `unwrap()` + pub static ref LIGHTHOUSE_JSON_CLIENT_VERSION: JsonClientVersionV1 = JsonClientVersionV1 { + code: ClientCode::Lighthouse.to_string(), + name: "Lighthouse".to_string(), + version: VERSION.replace("Lighthouse/", ""), + commit: COMMIT_PREFIX.to_string(), + }; +} + /// Contains methods to convert arbitrary bytes to an ETH2 deposit contract object. pub mod deposit_log { use ssz::Decode; @@ -546,22 +565,21 @@ pub mod deposit_methods { } } -#[derive(Clone, Debug)] -pub struct CapabilitiesCacheEntry { - engine_capabilities: EngineCapabilities, - fetch_time: Instant, +pub struct CachedResponse { + pub data: T, + pub fetch_time: Instant, } -impl CapabilitiesCacheEntry { - pub fn new(engine_capabilities: EngineCapabilities) -> Self { +impl CachedResponse { + pub fn new(data: T) -> Self { Self { - engine_capabilities, + data, fetch_time: Instant::now(), } } - pub fn engine_capabilities(&self) -> EngineCapabilities { - self.engine_capabilities + pub fn data(&self) -> T { + self.data.clone() } pub fn age(&self) -> Duration { @@ -578,7 +596,8 @@ pub struct HttpJsonRpc { pub client: Client, pub url: SensitiveUrl, pub execution_timeout_multiplier: u32, - pub engine_capabilities_cache: Mutex>, + pub engine_capabilities_cache: Mutex>>, + pub engine_version_cache: Mutex>>>, auth: Option, } @@ -592,6 +611,7 @@ impl HttpJsonRpc { url, execution_timeout_multiplier: execution_timeout_multiplier.unwrap_or(1), engine_capabilities_cache: Mutex::new(None), + engine_version_cache: Mutex::new(None), auth: None, }) } @@ -606,6 +626,7 @@ impl HttpJsonRpc { url, execution_timeout_multiplier: execution_timeout_multiplier.unwrap_or(1), engine_capabilities_cache: Mutex::new(None), + engine_version_cache: Mutex::new(None), auth: Some(auth), }) } @@ -1056,6 +1077,7 @@ impl HttpJsonRpc { get_payload_v1: capabilities.contains(ENGINE_GET_PAYLOAD_V1), get_payload_v2: capabilities.contains(ENGINE_GET_PAYLOAD_V2), get_payload_v3: capabilities.contains(ENGINE_GET_PAYLOAD_V3), + get_client_version_v1: capabilities.contains(ENGINE_GET_CLIENT_VERSION_V1), }) } @@ -1078,15 +1100,78 @@ impl HttpJsonRpc { ) -> Result { let mut lock = self.engine_capabilities_cache.lock().await; - if let Some(lock) = lock.as_ref().filter(|entry| !entry.older_than(age_limit)) { - Ok(lock.engine_capabilities()) + if let Some(lock) = lock + .as_ref() + .filter(|cached_response| !cached_response.older_than(age_limit)) + { + Ok(lock.data()) } else { let engine_capabilities = self.exchange_capabilities().await?; - *lock = Some(CapabilitiesCacheEntry::new(engine_capabilities)); + *lock = Some(CachedResponse::new(engine_capabilities)); Ok(engine_capabilities) } } + /// This method fetches the response from the engine without checking + /// any caches or storing the result in the cache. It is better to use + /// `get_engine_version(Some(Duration::ZERO))` if you want to force + /// fetching from the EE as this will cache the result. + pub async fn get_client_version_v1(&self) -> Result, Error> { + let params = json!([*LIGHTHOUSE_JSON_CLIENT_VERSION]); + + let response: Vec = self + .rpc_request( + ENGINE_GET_CLIENT_VERSION_V1, + params, + ENGINE_GET_CLIENT_VERSION_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + + response + .into_iter() + .map(TryInto::try_into) + .collect::, _>>() + .map_err(Error::InvalidClientVersion) + } + + pub async fn clear_engine_version_cache(&self) { + *self.engine_version_cache.lock().await = None; + } + + /// Returns the execution engine version resulting from a call to + /// engine_getClientVersionV1. If the version cache is not populated, or if it + /// is populated with a cached result of age >= `age_limit`, this method will + /// fetch the result from the execution engine and populate the cache before + /// returning it. Otherwise it will return the cached result from an earlier + /// call. + /// + /// Set `age_limit` to `None` to always return the cached result + /// Set `age_limit` to `Some(Duration::ZERO)` to force fetching from EE + pub async fn get_engine_version( + &self, + age_limit: Option, + ) -> Result, Error> { + // check engine capabilities first (avoids holding two locks at once) + let engine_capabilities = self.get_engine_capabilities(None).await?; + if !engine_capabilities.get_client_version_v1 { + // We choose an empty vec to denote that this method is not + // supported instead of an error since this method is optional + // & we don't want to log a warning and concern the user + return Ok(vec![]); + } + let mut lock = self.engine_version_cache.lock().await; + if let Some(lock) = lock + .as_ref() + .filter(|cached_response| !cached_response.older_than(age_limit)) + { + Ok(lock.data()) + } else { + let engine_version = self.get_client_version_v1().await?; + *lock = Some(CachedResponse::new(engine_version.clone())); + Ok(engine_version) + } + } + // automatically selects the latest version of // new_payload that the execution engine supports pub async fn new_payload( diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 9f2387ae314..d784aa4cd9c 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -747,3 +747,36 @@ pub mod serde_logs_bloom { .map_err(|e| serde::de::Error::custom(format!("invalid logs bloom: {:?}", e))) } } + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct JsonClientVersionV1 { + pub code: String, + pub name: String, + pub version: String, + pub commit: String, +} + +impl From for JsonClientVersionV1 { + fn from(client_version: ClientVersionV1) -> Self { + Self { + code: client_version.code.to_string(), + name: client_version.name, + version: client_version.version, + commit: client_version.commit.to_string(), + } + } +} + +impl TryFrom for ClientVersionV1 { + type Error = String; + + fn try_from(json: JsonClientVersionV1) -> Result { + Ok(Self { + code: json.code.try_into()?, + name: json.name, + version: json.version, + commit: json.commit.try_into()?, + }) + } +} diff --git a/beacon_node/execution_layer/src/engines.rs b/beacon_node/execution_layer/src/engines.rs index bc8e4e31404..75d0b872cef 100644 --- a/beacon_node/execution_layer/src/engines.rs +++ b/beacon_node/execution_layer/src/engines.rs @@ -4,7 +4,7 @@ use crate::engine_api::{ EngineCapabilities, Error as EngineApiError, ForkchoiceUpdatedResponse, PayloadAttributes, PayloadId, }; -use crate::HttpJsonRpc; +use crate::{ClientVersionV1, HttpJsonRpc}; use lru::LruCache; use slog::{debug, error, info, warn, Logger}; use std::future::Future; @@ -21,7 +21,7 @@ use types::ExecutionBlockHash; /// /// Since the size of each value is small (~800 bytes) a large number is used for safety. const PAYLOAD_ID_LRU_CACHE_SIZE: NonZeroUsize = new_non_zero_usize(512); -const CACHED_ENGINE_CAPABILITIES_AGE_LIMIT: Duration = Duration::from_secs(900); // 15 minutes +const CACHED_RESPONSE_AGE_LIMIT: Duration = Duration::from_secs(900); // 15 minutes /// Stores the remembered state of a engine. #[derive(Copy, Clone, PartialEq, Debug, Eq, Default)] @@ -34,11 +34,11 @@ enum EngineStateInternal { } #[derive(Copy, Clone, Debug, Default, Eq, PartialEq)] -enum CapabilitiesCacheAction { +enum ResponseCacheAction { #[default] None, - Update, - Clear, + Update, // Update cached responses + Clear, // Clear cached responses } /// A subset of the engine state to inform other services if the engine is online or offline. @@ -266,12 +266,12 @@ impl Engine { ); } state.update(EngineStateInternal::Synced); - (**state, CapabilitiesCacheAction::Update) + (**state, ResponseCacheAction::Update) } Err(EngineApiError::IsSyncing) => { let mut state = self.state.write().await; state.update(EngineStateInternal::Syncing); - (**state, CapabilitiesCacheAction::Update) + (**state, ResponseCacheAction::Update) } Err(EngineApiError::Auth(err)) => { error!( @@ -282,7 +282,7 @@ impl Engine { let mut state = self.state.write().await; state.update(EngineStateInternal::AuthFailed); - (**state, CapabilitiesCacheAction::Clear) + (**state, ResponseCacheAction::Clear) } Err(e) => { error!( @@ -293,28 +293,37 @@ impl Engine { let mut state = self.state.write().await; state.update(EngineStateInternal::Offline); - // need to clear the engine capabilities cache if we detect the - // execution engine is offline as it is likely the engine is being - // updated to a newer version with new capabilities - (**state, CapabilitiesCacheAction::Clear) + // need to clear cached responses if we detect the execution engine + // is offline as it is likely the engine is being updated to a newer + // version which might also have new capabilities + (**state, ResponseCacheAction::Clear) } }; // do this after dropping state lock guard to avoid holding two locks at once match cache_action { - CapabilitiesCacheAction::None => {} - CapabilitiesCacheAction::Update => { + ResponseCacheAction::None => {} + ResponseCacheAction::Update => { if let Err(e) = self - .get_engine_capabilities(Some(CACHED_ENGINE_CAPABILITIES_AGE_LIMIT)) + .get_engine_capabilities(Some(CACHED_RESPONSE_AGE_LIMIT)) .await { warn!(self.log, "Error during exchange capabilities"; "error" => ?e, ) + } else { + // no point in running this if there was an error fetching the capabilities + // as it will just result in an error again + let _ = self + .get_engine_version(Some(CACHED_RESPONSE_AGE_LIMIT)) + .await; } } - CapabilitiesCacheAction::Clear => self.api.clear_exchange_capabilties_cache().await, + ResponseCacheAction::Clear => { + self.api.clear_exchange_capabilties_cache().await; + self.api.clear_engine_version_cache().await; + } } debug!( @@ -340,6 +349,22 @@ impl Engine { self.api.get_engine_capabilities(age_limit).await } + /// Returns the execution engine version resulting from a call to + /// engine_clientVersionV1. If the version cache is not populated, or if it + /// is populated with a cached result of age >= `age_limit`, this method will + /// fetch the result from the execution engine and populate the cache before + /// returning it. Otherwise it will return the cached result from an earlier + /// call. + /// + /// Set `age_limit` to `None` to always return the cached result + /// Set `age_limit` to `Some(Duration::ZERO)` to force fetching from EE + pub async fn get_engine_version( + &self, + age_limit: Option, + ) -> Result, EngineApiError> { + self.api.get_engine_version(age_limit).await + } + /// Run `func` on the node regardless of the node's current state. /// /// ## Note diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 3e7bf7f561d..60f450a39de 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -165,6 +165,17 @@ impl From for Error { } } +impl From for Error { + fn from(e: EngineError) -> Self { + match e { + // This removes an unnecessary layer of indirection. + // TODO (mark): consider refactoring these error enums + EngineError::Api { error } => Error::ApiError(error), + _ => Error::EngineError(Box::new(e)), + } + } +} + pub enum BlockProposalContentsType { Full(BlockProposalContents>), Blinded(BlockProposalContents>), @@ -1526,8 +1537,26 @@ impl ExecutionLayer { self.engine() .request(|engine| engine.get_engine_capabilities(age_limit)) .await - .map_err(Box::new) - .map_err(Error::EngineError) + .map_err(Into::into) + } + + /// Returns the execution engine version resulting from a call to + /// engine_clientVersionV1. If the version cache is not populated, or if it + /// is populated with a cached result of age >= `age_limit`, this method will + /// fetch the result from the execution engine and populate the cache before + /// returning it. Otherwise it will return the cached result from an earlier + /// call. + /// + /// Set `age_limit` to `None` to always return the cached result + /// Set `age_limit` to `Some(Duration::ZERO)` to force fetching from EE + pub async fn get_engine_version( + &self, + age_limit: Option, + ) -> Result, Error> { + self.engine() + .request(|engine| engine.get_engine_version(age_limit)) + .await + .map_err(Into::into) } /// Used during block production to determine if the merge has been triggered. diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index e0ca07dcc6e..c16bd43c82c 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -1,7 +1,7 @@ use super::Context; use crate::engine_api::{http::*, *}; use crate::json_structures::*; -use crate::test_utils::DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI; +use crate::test_utils::{DEFAULT_CLIENT_VERSION, DEFAULT_MOCK_EL_PAYLOAD_VALUE_WEI}; use serde::{de::DeserializeOwned, Deserialize}; use serde_json::Value as JsonValue; use std::sync::Arc; @@ -528,6 +528,9 @@ pub async fn handle_rpc( let engine_capabilities = ctx.engine_capabilities.read(); Ok(serde_json::to_value(engine_capabilities.to_response()).unwrap()) } + ENGINE_GET_CLIENT_VERSION_V1 => { + Ok(serde_json::to_value([DEFAULT_CLIENT_VERSION.clone()]).unwrap()) + } ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1 => { #[derive(Deserialize)] #[serde(transparent)] diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index 43a6ee8ac22..a6d47995af8 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -4,11 +4,13 @@ use crate::engine_api::auth::JwtKey; use crate::engine_api::{ auth::Auth, http::JSONRPC_VERSION, ExecutionBlock, PayloadStatusV1, PayloadStatusV1Status, }; +use crate::json_structures::JsonClientVersionV1; use bytes::Bytes; use environment::null_logger; use execution_block_generator::PoWBlock; use handle_rpc::handle_rpc; use kzg::Kzg; +use lazy_static::lazy_static; use parking_lot::{Mutex, RwLock, RwLockWriteGuard}; use serde::{Deserialize, Serialize}; use serde_json::json; @@ -49,8 +51,18 @@ pub const DEFAULT_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities { get_payload_v1: true, get_payload_v2: true, get_payload_v3: true, + get_client_version_v1: true, }; +lazy_static! { + pub static ref DEFAULT_CLIENT_VERSION: JsonClientVersionV1 = JsonClientVersionV1 { + code: "MC".to_string(), // "mock client" + name: "Mock Execution Client".to_string(), + version: "0.1.0".to_string(), + commit: "0xabcdef01".to_string(), + }; +} + mod execution_block_generator; mod handle_rpc; mod hook; diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index d31b576989c..9a1d7df124c 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -3,6 +3,7 @@ use beacon_chain::chain_config::{ DEFAULT_RE_ORG_HEAD_THRESHOLD, DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION, DEFAULT_RE_ORG_PARENT_THRESHOLD, }; +use beacon_chain::graffiti_calculator::GraffitiOrigin; use beacon_chain::TrustedSetup; use clap::ArgMatches; use clap_utils::flags::DISABLE_MALLOC_TUNING_FLAG; @@ -17,7 +18,6 @@ use lighthouse_network::ListenAddress; use lighthouse_network::{multiaddr::Protocol, Enr, Multiaddr, NetworkConfig, PeerIdSerialized}; use sensitive_url::SensitiveUrl; use slog::{info, warn, Logger}; -use std::cmp; use std::cmp::max; use std::fmt::Debug; use std::fs; @@ -27,7 +27,8 @@ use std::num::NonZeroU16; use std::path::{Path, PathBuf}; use std::str::FromStr; use std::time::Duration; -use types::{Checkpoint, Epoch, EthSpec, Hash256, PublicKeyBytes, GRAFFITI_BYTES_LEN}; +use types::graffiti::GraffitiString; +use types::{Checkpoint, Epoch, EthSpec, Hash256, PublicKeyBytes}; /// Gets the fully-initialized global client. /// @@ -576,24 +577,16 @@ pub fn get_config( client_config.chain.genesis_backfill = true; } - let raw_graffiti = if let Some(graffiti) = cli_args.value_of("graffiti") { - if graffiti.len() > GRAFFITI_BYTES_LEN { - return Err(format!( - "Your graffiti is too long! {} bytes maximum!", - GRAFFITI_BYTES_LEN - )); - } - - graffiti.as_bytes() + let beacon_graffiti = if let Some(graffiti) = cli_args.value_of("graffiti") { + GraffitiOrigin::UserSpecified(GraffitiString::from_str(graffiti)?.into()) } else if cli_args.is_present("private") { - b"" + // When 'private' flag is present, use a zero-initialized bytes array. + GraffitiOrigin::UserSpecified(GraffitiString::empty().into()) } else { - lighthouse_version::VERSION.as_bytes() + // Use the default lighthouse graffiti if no user-specified graffiti flags are present + GraffitiOrigin::default() }; - - let trimmed_graffiti_len = cmp::min(raw_graffiti.len(), GRAFFITI_BYTES_LEN); - client_config.graffiti.0[..trimmed_graffiti_len] - .copy_from_slice(&raw_graffiti[..trimmed_graffiti_len]); + client_config.beacon_graffiti = beacon_graffiti; if let Some(wss_checkpoint) = cli_args.value_of("wss-checkpoint") { let mut split = wss_checkpoint.split(':'); diff --git a/common/lighthouse_version/src/lib.rs b/common/lighthouse_version/src/lib.rs index 5387d322e96..985eaff1b59 100644 --- a/common/lighthouse_version/src/lib.rs +++ b/common/lighthouse_version/src/lib.rs @@ -21,6 +21,24 @@ pub const VERSION: &str = git_version!( fallback = "Lighthouse/v5.1.3" ); +/// Returns the first eight characters of the latest commit hash for this build. +/// +/// No indication is given if the tree is dirty. This is part of the standard +/// for reporting the client version to the execution engine. +pub const COMMIT_PREFIX: &str = git_version!( + args = [ + "--always", + "--abbrev=8", + // NOTE: using --match instead of --exclude for compatibility with old Git + "--match=thiswillnevermatchlol" + ], + prefix = "", + suffix = "", + cargo_prefix = "", + cargo_suffix = "", + fallback = "00000000" +); + /// Returns `VERSION`, but with platform information appended to the end. /// /// ## Example diff --git a/consensus/types/src/graffiti.rs b/consensus/types/src/graffiti.rs index bd4abe37d8f..c7a0081c9f7 100644 --- a/consensus/types/src/graffiti.rs +++ b/consensus/types/src/graffiti.rs @@ -47,6 +47,12 @@ impl Into<[u8; GRAFFITI_BYTES_LEN]> for Graffiti { #[serde(transparent)] pub struct GraffitiString(String); +impl GraffitiString { + pub fn empty() -> Self { + Self(String::new()) + } +} + impl FromStr for GraffitiString { type Err = String; diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 9983717cb1e..9f1e3d235d9 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -5,9 +5,11 @@ use beacon_node::beacon_chain::chain_config::{ DisallowedReOrgOffsets, DEFAULT_RE_ORG_CUTOFF_DENOMINATOR, DEFAULT_RE_ORG_HEAD_THRESHOLD, DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION, }; +use beacon_node::beacon_chain::graffiti_calculator::GraffitiOrigin; use beacon_processor::BeaconProcessorConfig; use eth1::Eth1Endpoint; use lighthouse_network::PeerId; +use lighthouse_version; use std::fs::File; use std::io::{Read, Write}; use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; @@ -296,13 +298,36 @@ fn graffiti_flag() { .flag("graffiti", Some("nice-graffiti")) .run_with_zero_port() .with_config(|config| { + assert!(matches!( + config.beacon_graffiti, + GraffitiOrigin::UserSpecified(_) + )); assert_eq!( - config.graffiti.to_string(), - "0x6e6963652d677261666669746900000000000000000000000000000000000000" + config.beacon_graffiti.graffiti().to_string(), + "0x6e6963652d677261666669746900000000000000000000000000000000000000", ); }); } +#[test] +fn default_graffiti() { + use types::GRAFFITI_BYTES_LEN; + // test default graffiti when no graffiti flags are provided + CommandLineTest::new() + .run_with_zero_port() + .with_config(|config| { + assert!(matches!( + config.beacon_graffiti, + GraffitiOrigin::Calculated(_) + )); + let version_bytes = lighthouse_version::VERSION.as_bytes(); + let trimmed_len = std::cmp::min(version_bytes.len(), GRAFFITI_BYTES_LEN); + let mut bytes = [0u8; GRAFFITI_BYTES_LEN]; + bytes[..trimmed_len].copy_from_slice(&version_bytes[..trimmed_len]); + assert_eq!(config.beacon_graffiti.graffiti().0, bytes); + }); +} + #[test] fn trusted_peers_flag() { let peers = vec![PeerId::random(), PeerId::random()]; @@ -1201,7 +1226,17 @@ fn private_flag() { CommandLineTest::new() .flag("private", None) .run_with_zero_port() - .with_config(|config| assert!(config.network.private)); + .with_config(|config| { + assert!(config.network.private); + assert!(matches!( + config.beacon_graffiti, + GraffitiOrigin::UserSpecified(_) + )); + assert_eq!( + config.beacon_graffiti.graffiti().to_string(), + "0x0000000000000000000000000000000000000000000000000000000000000000".to_string(), + ); + }); } #[test] fn zero_ports_flag() { From dd340eebdc304d79bc5c391d293d5e773fc5089e Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 25 Apr 2024 13:11:01 -0400 Subject: [PATCH 29/37] Fix execution integration tests (#5647) * update waiting status * revert to old nethermind version --- testing/execution_engine_integration/src/test_rig.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/execution_engine_integration/src/test_rig.rs b/testing/execution_engine_integration/src/test_rig.rs index 0103f7074b5..c7d5e704524 100644 --- a/testing/execution_engine_integration/src/test_rig.rs +++ b/testing/execution_engine_integration/src/test_rig.rs @@ -180,7 +180,7 @@ impl TestRig { // Run the routine to check for online nodes. pair.execution_layer.watchdog_task().await; - if pair.execution_layer.is_synced().await { + if !pair.execution_layer.is_offline_or_erroring().await { break; } else if start_instant + EXECUTION_ENGINE_START_TIMEOUT > Instant::now() { sleep(Duration::from_millis(500)).await; From 320345695d3a227571d5c1a2a976aa0feaf7ecd1 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 25 Apr 2024 14:33:03 -0400 Subject: [PATCH 30/37] Add electra presets to beacon API (#5630) * add presets to API * add extra fields to config spec in beacon API * remove unused * add mainnet presets for gnosis and fix minimal preset default values --- consensus/types/presets/gnosis/electra.yaml | 46 +++++++++++++++++++- consensus/types/presets/mainnet/electra.yaml | 44 ++++++++++++++++++- consensus/types/presets/minimal/electra.yaml | 44 ++++++++++++++++++- consensus/types/src/chain_spec.rs | 2 + consensus/types/src/config_and_preset.rs | 12 ++++- consensus/types/src/consts.rs | 3 ++ consensus/types/src/eth_spec.rs | 10 ++--- consensus/types/src/preset.rs | 43 ++++++++++++++++-- 8 files changed, 190 insertions(+), 14 deletions(-) diff --git a/consensus/types/presets/gnosis/electra.yaml b/consensus/types/presets/gnosis/electra.yaml index cafdcbbf8d3..72c626ded2f 100644 --- a/consensus/types/presets/gnosis/electra.yaml +++ b/consensus/types/presets/gnosis/electra.yaml @@ -1,3 +1,45 @@ -# Gnosis preset - Electra +# Mainnet preset - Electra -ELECTRA_PLACEHOLDER: 0 +# Gwei values +# --------------------------------------------------------------- +# 2**5 * 10**9 (= 32,000,000,000) Gwei +MIN_ACTIVATION_BALANCE: 32000000000 +# 2**11 * 10**9 (= 2,048,000,000,000) Gwei +MAX_EFFECTIVE_BALANCE_ELECTRA: 2048000000000 + +# State list lengths +# --------------------------------------------------------------- +# `uint64(2**27)` (= 134,217,728) +PENDING_BALANCE_DEPOSITS_LIMIT: 134217728 +# `uint64(2**27)` (= 134,217,728) +PENDING_PARTIAL_WITHDRAWALS_LIMIT: 134217728 +# `uint64(2**18)` (= 262,144) +PENDING_CONSOLIDATIONS_LIMIT: 262144 + +# Reward and penalty quotients +# --------------------------------------------------------------- +# `uint64(2**12)` (= 4,096) +MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA: 4096 +# `uint64(2**12)` (= 4,096) +WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: 4096 + +# # Max operations per block +# --------------------------------------------------------------- +# `uint64(2**0)` (= 1) +MAX_ATTESTER_SLASHINGS_ELECTRA: 1 +# `uint64(2**3)` (= 8) +MAX_ATTESTATIONS_ELECTRA: 8 +# `uint64(2**0)` (= 1) +MAX_CONSOLIDATIONS: 1 + +# Execution +# --------------------------------------------------------------- +# 2**13 (= 8192) receipts +MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 8192 +# 2**4 (= 16) withdrawal requests +MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 16 + +# Withdrawals processing +# --------------------------------------------------------------- +# 2**3 ( = 8) pending withdrawals +MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 8 diff --git a/consensus/types/presets/mainnet/electra.yaml b/consensus/types/presets/mainnet/electra.yaml index 64d8b97b631..72c626ded2f 100644 --- a/consensus/types/presets/mainnet/electra.yaml +++ b/consensus/types/presets/mainnet/electra.yaml @@ -1,3 +1,45 @@ # Mainnet preset - Electra -ELECTRA_PLACEHOLDER: 0 +# Gwei values +# --------------------------------------------------------------- +# 2**5 * 10**9 (= 32,000,000,000) Gwei +MIN_ACTIVATION_BALANCE: 32000000000 +# 2**11 * 10**9 (= 2,048,000,000,000) Gwei +MAX_EFFECTIVE_BALANCE_ELECTRA: 2048000000000 + +# State list lengths +# --------------------------------------------------------------- +# `uint64(2**27)` (= 134,217,728) +PENDING_BALANCE_DEPOSITS_LIMIT: 134217728 +# `uint64(2**27)` (= 134,217,728) +PENDING_PARTIAL_WITHDRAWALS_LIMIT: 134217728 +# `uint64(2**18)` (= 262,144) +PENDING_CONSOLIDATIONS_LIMIT: 262144 + +# Reward and penalty quotients +# --------------------------------------------------------------- +# `uint64(2**12)` (= 4,096) +MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA: 4096 +# `uint64(2**12)` (= 4,096) +WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: 4096 + +# # Max operations per block +# --------------------------------------------------------------- +# `uint64(2**0)` (= 1) +MAX_ATTESTER_SLASHINGS_ELECTRA: 1 +# `uint64(2**3)` (= 8) +MAX_ATTESTATIONS_ELECTRA: 8 +# `uint64(2**0)` (= 1) +MAX_CONSOLIDATIONS: 1 + +# Execution +# --------------------------------------------------------------- +# 2**13 (= 8192) receipts +MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 8192 +# 2**4 (= 16) withdrawal requests +MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 16 + +# Withdrawals processing +# --------------------------------------------------------------- +# 2**3 ( = 8) pending withdrawals +MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 8 diff --git a/consensus/types/presets/minimal/electra.yaml b/consensus/types/presets/minimal/electra.yaml index 3baa7fa8161..11aa5e1f50e 100644 --- a/consensus/types/presets/minimal/electra.yaml +++ b/consensus/types/presets/minimal/electra.yaml @@ -1,3 +1,45 @@ # Minimal preset - Electra -ELECTRA_PLACEHOLDER: 0 +# Gwei values +# --------------------------------------------------------------- +# 2**5 * 10**9 (= 32,000,000,000) Gwei +MIN_ACTIVATION_BALANCE: 32000000000 +# 2**11 * 10**9 (= 2,048,000,000,000) Gwei +MAX_EFFECTIVE_BALANCE_ELECTRA: 2048000000000 + +# State list lengths +# --------------------------------------------------------------- +# `uint64(2**27)` (= 134,217,728) +PENDING_BALANCE_DEPOSITS_LIMIT: 134217728 +# [customized] `uint64(2**6)` (= 64) +PENDING_PARTIAL_WITHDRAWALS_LIMIT: 64 +# [customized] `uint64(2**6)` (= 64) +PENDING_CONSOLIDATIONS_LIMIT: 64 + +# Reward and penalty quotients +# --------------------------------------------------------------- +# `uint64(2**12)` (= 4,096) +MIN_SLASHING_PENALTY_QUOTIENT_ELECTRA: 4096 +# `uint64(2**12)` (= 4,096) +WHISTLEBLOWER_REWARD_QUOTIENT_ELECTRA: 4096 + +# # Max operations per block +# --------------------------------------------------------------- +# `uint64(2**0)` (= 1) +MAX_ATTESTER_SLASHINGS_ELECTRA: 1 +# `uint64(2**3)` (= 8) +MAX_ATTESTATIONS_ELECTRA: 8 +# `uint64(2**0)` (= 1) +MAX_CONSOLIDATIONS: 1 + +# Execution +# --------------------------------------------------------------- +# [customized] +MAX_DEPOSIT_RECEIPTS_PER_PAYLOAD: 4 +# [customized] 2**1 (= 2) withdrawal requests +MAX_WITHDRAWAL_REQUESTS_PER_PAYLOAD: 2 + +# Withdrawals processing +# --------------------------------------------------------------- +# 2**0 ( = 1) pending withdrawals +MAX_PENDING_PARTIALS_PER_WITHDRAWALS_SWEEP: 1 diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index e4f27d6873c..c55fe77a1b9 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -850,6 +850,8 @@ impl ChainSpec { // Electra electra_fork_version: [0x05, 0x00, 0x00, 0x01], electra_fork_epoch: None, + max_pending_partials_per_withdrawals_sweep: u64::checked_pow(2, 0) + .expect("pow does not overflow"), // Other network_id: 2, // lighthouse testnet network id deposit_chain_id: 5, diff --git a/consensus/types/src/config_and_preset.rs b/consensus/types/src/config_and_preset.rs index f2a6e616daa..6fc6e0642ea 100644 --- a/consensus/types/src/config_and_preset.rs +++ b/consensus/types/src/config_and_preset.rs @@ -1,6 +1,6 @@ use crate::{ - consts::altair, AltairPreset, BasePreset, BellatrixPreset, CapellaPreset, ChainSpec, Config, - DenebPreset, ElectraPreset, EthSpec, ForkName, + consts::altair, consts::deneb, AltairPreset, BasePreset, BellatrixPreset, CapellaPreset, + ChainSpec, Config, DenebPreset, ElectraPreset, EthSpec, ForkName, }; use maplit::hashmap; use serde::{Deserialize, Serialize}; @@ -100,6 +100,7 @@ pub fn get_extra_fields(spec: &ChainSpec) -> HashMap { let u8_hex = |v: u8| hex_string(&v.to_le_bytes()); hashmap! { "bls_withdrawal_prefix".to_uppercase() => u8_hex(spec.bls_withdrawal_prefix_byte), + "eth1_address_withdrawal_prefix".to_uppercase() => u8_hex(spec.eth1_address_withdrawal_prefix_byte), "domain_beacon_proposer".to_uppercase() => u32_hex(spec.domain_beacon_proposer), "domain_beacon_attester".to_uppercase() => u32_hex(spec.domain_beacon_attester), "domain_randao".to_uppercase()=> u32_hex(spec.domain_randao), @@ -119,6 +120,13 @@ pub fn get_extra_fields(spec: &ChainSpec) -> HashMap { altair::SYNC_COMMITTEE_SUBNET_COUNT.to_string().into(), "target_aggregators_per_sync_subcommittee".to_uppercase() => altair::TARGET_AGGREGATORS_PER_SYNC_SUBCOMMITTEE.to_string().into(), + // Deneb + "versioned_hash_version_kzg".to_uppercase() => deneb::VERSIONED_HASH_VERSION_KZG.to_string().into(), + // Electra + "compounding_withdrawal_prefix".to_uppercase() => u8_hex(spec.compounding_withdrawal_prefix_byte), + "unset_deposit_receipts_start_index".to_uppercase() => spec.unset_deposit_receipts_start_index.to_string().into(), + "full_exit_request_amount".to_uppercase() => spec.full_exit_request_amount.to_string().into(), + "domain_consolidation".to_uppercase()=> u32_hex(spec.domain_consolidation), } } diff --git a/consensus/types/src/consts.rs b/consensus/types/src/consts.rs index a9377bc3e00..31d4c5eac59 100644 --- a/consensus/types/src/consts.rs +++ b/consensus/types/src/consts.rs @@ -22,3 +22,6 @@ pub mod altair { pub mod merge { pub const INTERVALS_PER_SLOT: u64 = 3; } +pub mod deneb { + pub use crate::VERSIONED_HASH_VERSION_KZG; +} diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index a700c5e9abb..6749a3f3a2b 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -418,6 +418,10 @@ impl EthSpec for MinimalEthSpec { type BytesPerBlob = U131072; type MaxBlobCommitmentsPerBlock = U16; type KzgCommitmentInclusionProofDepth = U9; + type PendingPartialWithdrawalsLimit = U64; + type PendingConsolidationsLimit = U64; + type MaxDepositReceiptsPerPayload = U4; + type MaxWithdrawalRequestsPerPayload = U2; params_from_eth_spec!(MainnetEthSpec { JustificationBitsLength, @@ -442,13 +446,9 @@ impl EthSpec for MinimalEthSpec { MaxBlobsPerBlock, BytesPerFieldElement, PendingBalanceDepositsLimit, - PendingPartialWithdrawalsLimit, - PendingConsolidationsLimit, MaxConsolidations, - MaxDepositReceiptsPerPayload, MaxAttesterSlashingsElectra, - MaxAttestationsElectra, - MaxWithdrawalRequestsPerPayload + MaxAttestationsElectra }); fn default_spec() -> ChainSpec { diff --git a/consensus/types/src/preset.rs b/consensus/types/src/preset.rs index 626a45d971d..f4008d62e1d 100644 --- a/consensus/types/src/preset.rs +++ b/consensus/types/src/preset.rs @@ -230,13 +230,50 @@ impl DenebPreset { #[serde(rename_all = "UPPERCASE")] pub struct ElectraPreset { #[serde(with = "serde_utils::quoted_u64")] - pub electra_placeholder: u64, + pub min_activation_balance: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub max_effective_balance_electra: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub min_slashing_penalty_quotient_electra: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub whistleblower_reward_quotient_electra: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub max_pending_partials_per_withdrawals_sweep: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub pending_balance_deposits_limit: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub pending_partial_withdrawals_limit: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub pending_consolidations_limit: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub max_consolidations: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub max_deposit_receipts_per_payload: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub max_attester_slashings_electra: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub max_attestations_electra: u64, + #[serde(with = "serde_utils::quoted_u64")] + pub max_withdrawal_requests_per_payload: u64, } impl ElectraPreset { - pub fn from_chain_spec(_spec: &ChainSpec) -> Self { + pub fn from_chain_spec(spec: &ChainSpec) -> Self { Self { - electra_placeholder: 0, + min_activation_balance: spec.min_activation_balance, + max_effective_balance_electra: spec.max_effective_balance_electra, + min_slashing_penalty_quotient_electra: spec.min_slashing_penalty_quotient_electra, + whistleblower_reward_quotient_electra: spec.whistleblower_reward_quotient_electra, + max_pending_partials_per_withdrawals_sweep: spec + .max_pending_partials_per_withdrawals_sweep, + pending_balance_deposits_limit: E::pending_balance_deposits_limit() as u64, + pending_partial_withdrawals_limit: E::pending_partial_withdrawals_limit() as u64, + pending_consolidations_limit: E::pending_consolidations_limit() as u64, + max_consolidations: E::max_consolidations() as u64, + max_deposit_receipts_per_payload: E::max_deposit_receipts_per_payload() as u64, + max_attester_slashings_electra: E::max_attester_slashings_electra() as u64, + max_attestations_electra: E::max_attestations_electra() as u64, + max_withdrawal_requests_per_payload: E::max_withdrawal_requests_per_payload() as u64, } } } From 13f94ef0f324d98d313c6e13e1833f1825f245e1 Mon Sep 17 00:00:00 2001 From: Mac L Date: Fri, 26 Apr 2024 06:19:41 +1000 Subject: [PATCH 31/37] Rename `Merge` to `Bellatrix` (#5601) * Rename Merge to Bellatrix * Remove tree-hash-cache which got readded from the rebase --- Makefile | 2 +- .../beacon_chain/src/attestation_rewards.rs | 2 +- .../src/attestation_verification.rs | 4 +- .../beacon_chain/src/beacon_block_streamer.rs | 6 +- beacon_node/beacon_chain/src/beacon_chain.rs | 16 +-- ...ge_readiness.rs => bellatrix_readiness.rs} | 34 +++--- .../beacon_chain/src/capella_readiness.rs | 2 +- .../overflow_lru_cache.rs | 10 +- .../beacon_chain/src/deneb_readiness.rs | 2 +- .../beacon_chain/src/electra_readiness.rs | 2 +- .../beacon_chain/src/execution_payload.rs | 14 +-- beacon_node/beacon_chain/src/lib.rs | 2 +- beacon_node/beacon_chain/src/test_utils.rs | 4 +- .../tests/attestation_verification.rs | 20 ++-- .../tests/{merge.rs => bellatrix.rs} | 42 ++++---- beacon_node/beacon_chain/tests/capella.rs | 40 +++---- beacon_node/beacon_chain/tests/main.rs | 2 +- .../beacon_chain/tests/validator_monitor.rs | 4 +- beacon_node/client/src/notifier.rs | 24 ++--- beacon_node/execution_layer/src/block_hash.rs | 2 +- beacon_node/execution_layer/src/engine_api.rs | 69 ++++++------ .../execution_layer/src/engine_api/http.rs | 26 ++--- .../src/engine_api/json_structures.rs | 14 +-- .../src/engine_api/new_payload_request.rs | 40 ++++--- beacon_node/execution_layer/src/lib.rs | 44 ++++---- .../test_utils/execution_block_generator.rs | 22 ++-- .../src/test_utils/handle_rpc.rs | 6 +- .../src/test_utils/mock_builder.rs | 30 +++--- .../src/test_utils/mock_execution_layer.rs | 4 +- .../http_api/src/build_block_contents.rs | 2 +- beacon_node/http_api/src/builder_states.rs | 2 +- beacon_node/http_api/src/lib.rs | 2 +- beacon_node/http_api/src/publish_blocks.rs | 10 +- beacon_node/lighthouse_network/src/config.rs | 6 +- .../lighthouse_network/src/rpc/codec/base.rs | 6 +- .../src/rpc/codec/ssz_snappy.rs | 86 ++++++++------- .../lighthouse_network/src/rpc/protocol.rs | 40 ++++--- .../lighthouse_network/src/types/pubsub.rs | 10 +- .../lighthouse_network/src/types/topics.rs | 2 +- .../lighthouse_network/tests/common.rs | 6 +- .../lighthouse_network/tests/rpc_tests.rs | 59 +++++----- .../network_beacon_processor/rpc_methods.rs | 7 +- .../store/src/impls/execution_payload.rs | 9 +- beacon_node/store/src/partial_beacon_state.rs | 30 +++--- common/eth2/src/types.rs | 12 +-- .../gnosis/config.yaml | 2 +- .../holesky/config.yaml | 2 +- .../sepolia/config.yaml | 2 +- common/slot_clock/src/lib.rs | 2 +- consensus/fork_choice/src/fork_choice.rs | 4 +- .../common/get_attestation_participation.rs | 2 +- .../src/common/slash_validator.rs | 2 +- consensus/state_processing/src/genesis.rs | 6 +- .../src/per_block_processing.rs | 10 +- .../process_operations.rs | 2 +- .../per_block_processing/signature_sets.rs | 2 +- .../verify_attestation.rs | 2 +- .../src/per_epoch_processing.rs | 2 +- .../src/per_slot_processing.rs | 2 +- consensus/state_processing/src/upgrade.rs | 4 +- .../src/upgrade/{merge.rs => bellatrix.rs} | 10 +- .../state_processing/src/upgrade/capella.rs | 4 +- consensus/types/src/beacon_block.rs | 24 ++--- consensus/types/src/beacon_block_body.rs | 45 ++++---- consensus/types/src/beacon_state.rs | 50 ++++----- .../progressive_balances_cache.rs | 2 +- consensus/types/src/beacon_state/tests.rs | 6 +- consensus/types/src/builder_bid.rs | 14 +-- consensus/types/src/chain_spec.rs | 22 ++-- consensus/types/src/consts.rs | 2 +- consensus/types/src/eth_spec.rs | 2 +- consensus/types/src/execution_payload.rs | 16 +-- .../types/src/execution_payload_header.rs | 28 +++-- consensus/types/src/fork_context.rs | 6 +- consensus/types/src/fork_name.rs | 28 ++--- .../types/src/fork_versioned_response.rs | 9 +- consensus/types/src/lib.rs | 28 ++--- consensus/types/src/light_client_bootstrap.rs | 6 +- .../types/src/light_client_finality_update.rs | 6 +- consensus/types/src/light_client_header.rs | 8 +- .../src/light_client_optimistic_update.rs | 20 ++-- consensus/types/src/light_client_update.rs | 4 +- consensus/types/src/payload.rs | 102 ++++++++++-------- consensus/types/src/signed_beacon_block.rs | 49 +++++---- consensus/types/src/voluntary_exit.rs | 2 +- lcli/src/create_payload_header.rs | 11 +- lcli/src/main.rs | 2 +- lcli/src/new_testnet.rs | 18 ++-- lcli/src/parse_ssz.rs | 12 ++- .../environment/tests/testnet_dir/config.yaml | 2 +- lighthouse/tests/beacon_node.rs | 14 +-- testing/ef_tests/src/cases/common.rs | 4 +- .../ef_tests/src/cases/epoch_processing.rs | 16 +-- testing/ef_tests/src/cases/fork.rs | 4 +- .../src/cases/merkle_proof_validity.rs | 2 +- testing/ef_tests/src/cases/operations.rs | 23 ++-- testing/ef_tests/src/cases/transition.rs | 2 +- testing/ef_tests/src/handler.rs | 8 +- testing/ef_tests/src/type_name.rs | 6 +- testing/ef_tests/tests/tests.rs | 34 +++--- testing/web3signer_tests/src/lib.rs | 41 ++++--- .../src/signing_method/web3signer.rs | 2 +- validator_client/src/validator_store.rs | 2 +- watch/src/database/mod.rs | 2 +- 104 files changed, 808 insertions(+), 714 deletions(-) rename beacon_node/beacon_chain/src/{merge_readiness.rs => bellatrix_readiness.rs} (91%) rename beacon_node/beacon_chain/tests/{merge.rs => bellatrix.rs} (81%) rename consensus/state_processing/src/upgrade/{merge.rs => bellatrix.rs} (90%) diff --git a/Makefile b/Makefile index 4072ab1e6d8..12d33cc3a8b 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ PROFILE ?= release # List of all hard forks. This list is used to set env variables for several tests so that # they run for different forks. -FORKS=phase0 altair merge capella deneb electra +FORKS=phase0 altair bellatrix capella deneb electra # Extra flags for Cargo CARGO_INSTALL_EXTRA_FLAGS?= diff --git a/beacon_node/beacon_chain/src/attestation_rewards.rs b/beacon_node/beacon_chain/src/attestation_rewards.rs index 491b7ef7da9..d48a83130e6 100644 --- a/beacon_node/beacon_chain/src/attestation_rewards.rs +++ b/beacon_node/beacon_chain/src/attestation_rewards.rs @@ -54,7 +54,7 @@ impl BeaconChain { match state { BeaconState::Base(_) => self.compute_attestation_rewards_base(state, validators), BeaconState::Altair(_) - | BeaconState::Merge(_) + | BeaconState::Bellatrix(_) | BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) => self.compute_attestation_rewards_altair(state, validators), diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index f3bde8678e1..471c43d94f8 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -1063,7 +1063,9 @@ pub fn verify_propagation_slot_range( let current_fork = spec.fork_name_at_slot::(slot_clock.now().ok_or(BeaconChainError::UnableToReadSlot)?); let earliest_permissible_slot = match current_fork { - ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => one_epoch_prior, + ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => { + one_epoch_prior + } // EIP-7045 ForkName::Deneb | ForkName::Electra => one_epoch_prior .epoch(E::slots_per_epoch()) diff --git a/beacon_node/beacon_chain/src/beacon_block_streamer.rs b/beacon_node/beacon_chain/src/beacon_block_streamer.rs index 4f413ce2a86..0c92b7c1f62 100644 --- a/beacon_node/beacon_chain/src/beacon_block_streamer.rs +++ b/beacon_node/beacon_chain/src/beacon_block_streamer.rs @@ -14,8 +14,8 @@ use types::{ SignedBlindedBeaconBlock, Slot, }; use types::{ - ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadElectra, ExecutionPayloadHeader, - ExecutionPayloadMerge, + ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella, ExecutionPayloadElectra, + ExecutionPayloadHeader, }; #[derive(PartialEq)] @@ -95,7 +95,7 @@ fn reconstruct_default_header_block( .map_err(BeaconChainError::InconsistentFork)?; let payload: ExecutionPayload = match fork { - ForkName::Merge => ExecutionPayloadMerge::default().into(), + ForkName::Bellatrix => ExecutionPayloadBellatrix::default().into(), ForkName::Capella => ExecutionPayloadCapella::default().into(), ForkName::Deneb => ExecutionPayloadDeneb::default().into(), ForkName::Electra => ExecutionPayloadElectra::default().into(), diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index dd1fe59b922..9c7ded313b6 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -2521,7 +2521,7 @@ impl BeaconChain { /// Check if the current slot is greater than or equal to the Capella fork epoch. pub fn current_slot_is_post_capella(&self) -> Result { let current_fork = self.spec.fork_name_at_slot::(self.slot()?); - if let ForkName::Base | ForkName::Altair | ForkName::Merge = current_fork { + if let ForkName::Base | ForkName::Altair | ForkName::Bellatrix = current_fork { Ok(false) } else { Ok(true) @@ -4823,7 +4823,7 @@ impl BeaconChain { // allows it to run concurrently with things like attestation packing. let prepare_payload_handle = match &state { BeaconState::Base(_) | BeaconState::Altair(_) => None, - BeaconState::Merge(_) + BeaconState::Bellatrix(_) | BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) => { @@ -5083,17 +5083,17 @@ impl BeaconChain { None, Uint256::zero(), ), - BeaconState::Merge(_) => { + BeaconState::Bellatrix(_) => { let block_proposal_contents = block_contents.ok_or(BlockProductionError::MissingExecutionPayload)?; let execution_payload_value = block_proposal_contents.block_value().to_owned(); ( - BeaconBlock::Merge(BeaconBlockMerge { + BeaconBlock::Bellatrix(BeaconBlockBellatrix { slot, proposer_index, parent_root, state_root: Hash256::zero(), - body: BeaconBlockBodyMerge { + body: BeaconBlockBodyBellatrix { randao_reveal, eth1_data, graffiti, @@ -5541,7 +5541,7 @@ impl BeaconChain { } else { let prepare_slot_fork = self.spec.fork_name_at_slot::(prepare_slot); let withdrawals = match prepare_slot_fork { - ForkName::Base | ForkName::Altair | ForkName::Merge => None, + ForkName::Base | ForkName::Altair | ForkName::Bellatrix => None, ForkName::Capella | ForkName::Deneb | ForkName::Electra => { let chain = self.clone(); self.spawn_blocking_handle( @@ -5556,7 +5556,7 @@ impl BeaconChain { }; let parent_beacon_block_root = match prepare_slot_fork { - ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => None, + ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => None, ForkName::Deneb | ForkName::Electra => { Some(pre_payload_attributes.parent_beacon_block_root) } @@ -6599,7 +6599,7 @@ impl BeaconChain { match fork_name { ForkName::Altair - | ForkName::Merge + | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb | ForkName::Electra => { diff --git a/beacon_node/beacon_chain/src/merge_readiness.rs b/beacon_node/beacon_chain/src/bellatrix_readiness.rs similarity index 91% rename from beacon_node/beacon_chain/src/merge_readiness.rs rename to beacon_node/beacon_chain/src/bellatrix_readiness.rs index 52a5ea912e0..bf9e8481261 100644 --- a/beacon_node/beacon_chain/src/merge_readiness.rs +++ b/beacon_node/beacon_chain/src/bellatrix_readiness.rs @@ -11,7 +11,7 @@ use types::*; /// The time before the Bellatrix fork when we will start issuing warnings about preparation. pub const SECONDS_IN_A_WEEK: u64 = 604800; -pub const MERGE_READINESS_PREPARATION_SECONDS: u64 = SECONDS_IN_A_WEEK * 2; +pub const BELLATRIX_READINESS_PREPARATION_SECONDS: u64 = SECONDS_IN_A_WEEK * 2; #[derive(Default, Debug, Serialize, Deserialize)] pub struct MergeConfig { @@ -81,7 +81,7 @@ impl MergeConfig { #[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] #[serde(tag = "type")] -pub enum MergeReadiness { +pub enum BellatrixReadiness { /// The node is ready, as far as we can tell. Ready { config: MergeConfig, @@ -94,29 +94,29 @@ pub enum MergeReadiness { NoExecutionEndpoint, } -impl fmt::Display for MergeReadiness { +impl fmt::Display for BellatrixReadiness { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - MergeReadiness::Ready { + BellatrixReadiness::Ready { config: params, current_difficulty, } => { write!( f, - "This node appears ready for the merge. \ + "This node appears ready for Bellatrix \ Params: {}, current_difficulty: {:?}", params, current_difficulty ) } - MergeReadiness::NotSynced => write!( + BellatrixReadiness::NotSynced => write!( f, "The execution endpoint is connected and configured, \ however it is not yet synced" ), - MergeReadiness::NoExecutionEndpoint => write!( + BellatrixReadiness::NoExecutionEndpoint => write!( f, "The --execution-endpoint flag is not specified, this is a \ - requirement for the merge" + requirement for Bellatrix" ), } } @@ -143,12 +143,12 @@ pub enum GenesisExecutionPayloadStatus { impl BeaconChain { /// Returns `true` if user has an EL configured, or if the Bellatrix fork has occurred or will - /// occur within `MERGE_READINESS_PREPARATION_SECONDS`. + /// occur within `BELLATRIX_READINESS_PREPARATION_SECONDS`. pub fn is_time_to_prepare_for_bellatrix(&self, current_slot: Slot) -> bool { if let Some(bellatrix_epoch) = self.spec.bellatrix_fork_epoch { let bellatrix_slot = bellatrix_epoch.start_slot(T::EthSpec::slots_per_epoch()); - let merge_readiness_preparation_slots = - MERGE_READINESS_PREPARATION_SECONDS / self.spec.seconds_per_slot; + let bellatrix_readiness_preparation_slots = + BELLATRIX_READINESS_PREPARATION_SECONDS / self.spec.seconds_per_slot; if self.execution_layer.is_some() { // The user has already configured an execution layer, start checking for readiness @@ -156,7 +156,7 @@ impl BeaconChain { true } else { // Return `true` if Bellatrix has happened or is within the preparation time. - current_slot + merge_readiness_preparation_slots > bellatrix_slot + current_slot + bellatrix_readiness_preparation_slots > bellatrix_slot } } else { // The Bellatrix fork epoch has not been defined yet, no need to prepare. @@ -164,22 +164,22 @@ impl BeaconChain { } } - /// Attempts to connect to the EL and confirm that it is ready for the merge. - pub async fn check_merge_readiness(&self, current_slot: Slot) -> MergeReadiness { + /// Attempts to connect to the EL and confirm that it is ready for Bellatrix. + pub async fn check_bellatrix_readiness(&self, current_slot: Slot) -> BellatrixReadiness { if let Some(el) = self.execution_layer.as_ref() { if !el.is_synced_for_notifier(current_slot).await { // The EL is not synced. - return MergeReadiness::NotSynced; + return BellatrixReadiness::NotSynced; } let params = MergeConfig::from_chainspec(&self.spec); let current_difficulty = el.get_current_difficulty().await.ok(); - MergeReadiness::Ready { + BellatrixReadiness::Ready { config: params, current_difficulty, } } else { // There is no EL configured. - MergeReadiness::NoExecutionEndpoint + BellatrixReadiness::NoExecutionEndpoint } } diff --git a/beacon_node/beacon_chain/src/capella_readiness.rs b/beacon_node/beacon_chain/src/capella_readiness.rs index cde71d462d7..88af7db0aaa 100644 --- a/beacon_node/beacon_chain/src/capella_readiness.rs +++ b/beacon_node/beacon_chain/src/capella_readiness.rs @@ -10,7 +10,7 @@ use std::time::Duration; use types::*; /// The time before the Capella fork when we will start issuing warnings about preparation. -use super::merge_readiness::SECONDS_IN_A_WEEK; +use super::bellatrix_readiness::SECONDS_IN_A_WEEK; pub const CAPELLA_READINESS_PREPARATION_SECONDS: u64 = SECONDS_IN_A_WEEK * 2; pub const ENGINE_CAPABILITIES_REFRESH_INTERVAL: u64 = 300; diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 3a8bccbad5c..91c776adc10 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -989,17 +989,17 @@ mod test { // go to bellatrix slot harness.extend_to_slot(bellatrix_fork_slot).await; - let merge_head = &harness.chain.head_snapshot().beacon_block; - assert!(merge_head.as_merge().is_ok()); - assert_eq!(merge_head.slot(), bellatrix_fork_slot); + let bellatrix_head = &harness.chain.head_snapshot().beacon_block; + assert!(bellatrix_head.as_bellatrix().is_ok()); + assert_eq!(bellatrix_head.slot(), bellatrix_fork_slot); assert!( - merge_head + bellatrix_head .message() .body() .execution_payload() .unwrap() .is_default_with_empty_roots(), - "Merge head is default payload" + "Bellatrix head is default payload" ); // Trigger the terminal PoW block. harness diff --git a/beacon_node/beacon_chain/src/deneb_readiness.rs b/beacon_node/beacon_chain/src/deneb_readiness.rs index 1ba6fe3ea6c..e11070a1f4f 100644 --- a/beacon_node/beacon_chain/src/deneb_readiness.rs +++ b/beacon_node/beacon_chain/src/deneb_readiness.rs @@ -10,7 +10,7 @@ use std::time::Duration; use types::*; /// The time before the Deneb fork when we will start issuing warnings about preparation. -use super::merge_readiness::SECONDS_IN_A_WEEK; +use super::bellatrix_readiness::SECONDS_IN_A_WEEK; pub const DENEB_READINESS_PREPARATION_SECONDS: u64 = SECONDS_IN_A_WEEK * 2; pub const ENGINE_CAPABILITIES_REFRESH_INTERVAL: u64 = 300; diff --git a/beacon_node/beacon_chain/src/electra_readiness.rs b/beacon_node/beacon_chain/src/electra_readiness.rs index 0e911bf37ec..42ee743fe6b 100644 --- a/beacon_node/beacon_chain/src/electra_readiness.rs +++ b/beacon_node/beacon_chain/src/electra_readiness.rs @@ -11,7 +11,7 @@ use std::time::Duration; use types::*; /// The time before the Electra fork when we will start issuing warnings about preparation. -use super::merge_readiness::SECONDS_IN_A_WEEK; +use super::bellatrix_readiness::SECONDS_IN_A_WEEK; pub const ELECTRA_READINESS_PREPARATION_SECONDS: u64 = SECONDS_IN_A_WEEK * 2; pub const ENGINE_CAPABILITIES_REFRESH_INTERVAL: u64 = 300; diff --git a/beacon_node/beacon_chain/src/execution_payload.rs b/beacon_node/beacon_chain/src/execution_payload.rs index b3804f0d23a..cbffe363422 100644 --- a/beacon_node/beacon_chain/src/execution_payload.rs +++ b/beacon_node/beacon_chain/src/execution_payload.rs @@ -336,7 +336,7 @@ pub fn validate_execution_payload_for_gossip( block: BeaconBlockRef<'_, T::EthSpec>, chain: &BeaconChain, ) -> Result<(), BlockError> { - // Only apply this validation if this is a merge beacon block. + // Only apply this validation if this is a Bellatrix beacon block. if let Ok(execution_payload) = block.body().execution_payload() { // This logic should match `is_execution_enabled`. We use only the execution block hash of // the parent here in order to avoid loading the parent state during gossip verification. @@ -385,7 +385,7 @@ pub fn validate_execution_payload_for_gossip( /// ## Errors /// /// Will return an error when using a pre-merge fork `state`. Ensure to only run this function -/// after the merge fork. +/// after the Bellatrix fork. /// /// ## Specification /// @@ -415,13 +415,13 @@ pub fn get_execution_payload( &BeaconState::Capella(_) | &BeaconState::Deneb(_) | &BeaconState::Electra(_) => { Some(get_expected_withdrawals(state, spec)?.into()) } - &BeaconState::Merge(_) => None, + &BeaconState::Bellatrix(_) => None, // These shouldn't happen but they're here to make the pattern irrefutable &BeaconState::Base(_) | &BeaconState::Altair(_) => None, }; let parent_beacon_block_root = match state { BeaconState::Deneb(_) | BeaconState::Electra(_) => Some(parent_block_root), - BeaconState::Merge(_) | BeaconState::Capella(_) => None, + BeaconState::Bellatrix(_) | BeaconState::Capella(_) => None, // These shouldn't happen but they're here to make the pattern irrefutable BeaconState::Base(_) | BeaconState::Altair(_) => None, }; @@ -457,12 +457,12 @@ pub fn get_execution_payload( /// Prepares an execution payload for inclusion in a block. /// -/// Will return `Ok(None)` if the merge fork has occurred, but a terminal block has not been found. +/// Will return `Ok(None)` if the Bellatrix fork has occurred, but a terminal block has not been found. /// /// ## Errors /// -/// Will return an error when using a pre-merge fork `state`. Ensure to only run this function -/// after the merge fork. +/// Will return an error when using a pre-Bellatrix fork `state`. Ensure to only run this function +/// after the Bellatrix fork. /// /// ## Specification /// diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 5ee7803436c..221bb8b2922 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -8,6 +8,7 @@ mod beacon_chain; mod beacon_fork_choice_store; pub mod beacon_proposer_cache; mod beacon_snapshot; +pub mod bellatrix_readiness; pub mod blob_verification; pub mod block_reward; mod block_times_cache; @@ -35,7 +36,6 @@ pub mod kzg_utils; pub mod light_client_finality_update_verification; pub mod light_client_optimistic_update_verification; mod light_client_server_cache; -pub mod merge_readiness; pub mod metrics; pub mod migrate; mod naive_aggregation_pool; diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index d3eeff8a7d9..8fbd5d575f9 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -880,7 +880,7 @@ where let block_contents: SignedBlockContentsTuple = match *signed_block { SignedBeaconBlock::Base(_) | SignedBeaconBlock::Altair(_) - | SignedBeaconBlock::Merge(_) + | SignedBeaconBlock::Bellatrix(_) | SignedBeaconBlock::Capella(_) => (signed_block, None), SignedBeaconBlock::Deneb(_) | SignedBeaconBlock::Electra(_) => { (signed_block, block_response.blob_items) @@ -944,7 +944,7 @@ where let block_contents: SignedBlockContentsTuple = match *signed_block { SignedBeaconBlock::Base(_) | SignedBeaconBlock::Altair(_) - | SignedBeaconBlock::Merge(_) + | SignedBeaconBlock::Bellatrix(_) | SignedBeaconBlock::Capella(_) => (signed_block, None), SignedBeaconBlock::Deneb(_) | SignedBeaconBlock::Electra(_) => { (signed_block, block_response.blob_items) diff --git a/beacon_node/beacon_chain/tests/attestation_verification.rs b/beacon_node/beacon_chain/tests/attestation_verification.rs index 3432604cc93..1463d1c5c15 100644 --- a/beacon_node/beacon_chain/tests/attestation_verification.rs +++ b/beacon_node/beacon_chain/tests/attestation_verification.rs @@ -336,7 +336,7 @@ impl GossipTester { pub fn earliest_valid_attestation_slot(&self) -> Slot { let offset = match self.harness.spec.fork_name_at_epoch(self.epoch()) { - ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => { // Subtract an additional slot since the harness will be exactly on the start of the // slot and the propagation tolerance will allow an extra slot. E::slots_per_epoch() + 1 @@ -1382,7 +1382,10 @@ async fn attestation_verification_use_head_state_fork() { .block_at_slot(pre_capella_slot, WhenSlotSkipped::Prev) .expect("should not error getting block at slot") .expect("should find block at slot"); - assert_eq!(pre_capella_block.fork_name(&spec).unwrap(), ForkName::Merge); + assert_eq!( + pre_capella_block.fork_name(&spec).unwrap(), + ForkName::Bellatrix + ); // Advance slot clock to Capella fork. harness.advance_slot(); @@ -1427,7 +1430,7 @@ async fn attestation_verification_use_head_state_fork() { // Scenario 2: other node forgot to update their node and signed attestations using bellatrix fork { let attesters = (VALIDATOR_COUNT / 2..VALIDATOR_COUNT).collect::>(); - let merge_fork = spec.fork_for_name(ForkName::Merge).unwrap(); + let bellatrix_fork = spec.fork_for_name(ForkName::Bellatrix).unwrap(); let committee_attestations = harness .make_unaggregated_attestations_with_opts( attesters.as_slice(), @@ -1436,7 +1439,7 @@ async fn attestation_verification_use_head_state_fork() { pre_capella_block.canonical_root().into(), first_capella_slot, MakeAttestationOptions { - fork: merge_fork, + fork: bellatrix_fork, limit: None, }, ) @@ -1483,7 +1486,10 @@ async fn aggregated_attestation_verification_use_head_state_fork() { .block_at_slot(pre_capella_slot, WhenSlotSkipped::Prev) .expect("should not error getting block at slot") .expect("should find block at slot"); - assert_eq!(pre_capella_block.fork_name(&spec).unwrap(), ForkName::Merge); + assert_eq!( + pre_capella_block.fork_name(&spec).unwrap(), + ForkName::Bellatrix + ); // Advance slot clock to Capella fork. harness.advance_slot(); @@ -1525,7 +1531,7 @@ async fn aggregated_attestation_verification_use_head_state_fork() { // Scenario 2: other node forgot to update their node and signed attestations using bellatrix fork { let attesters = (VALIDATOR_COUNT / 2..VALIDATOR_COUNT).collect::>(); - let merge_fork = spec.fork_for_name(ForkName::Merge).unwrap(); + let bellatrix_fork = spec.fork_for_name(ForkName::Bellatrix).unwrap(); let aggregates = harness .make_attestations_with_opts( attesters.as_slice(), @@ -1534,7 +1540,7 @@ async fn aggregated_attestation_verification_use_head_state_fork() { pre_capella_block.canonical_root().into(), first_capella_slot, MakeAttestationOptions { - fork: merge_fork, + fork: bellatrix_fork, limit: None, }, ) diff --git a/beacon_node/beacon_chain/tests/merge.rs b/beacon_node/beacon_chain/tests/bellatrix.rs similarity index 81% rename from beacon_node/beacon_chain/tests/merge.rs rename to beacon_node/beacon_chain/tests/bellatrix.rs index bff5c4523d1..027082c11c9 100644 --- a/beacon_node/beacon_chain/tests/merge.rs +++ b/beacon_node/beacon_chain/tests/bellatrix.rs @@ -71,9 +71,9 @@ async fn merge_with_terminal_block_hash_override() { .chain .head_snapshot() .beacon_block - .as_merge() + .as_bellatrix() .is_ok(), - "genesis block should be a merge block" + "genesis block should be a bellatrix block" ); let mut execution_payloads = vec![]; @@ -93,11 +93,11 @@ async fn merge_with_terminal_block_hash_override() { } #[tokio::test] -async fn base_altair_merge_with_terminal_block_after_fork() { +async fn base_altair_bellatrix_with_terminal_block_after_fork() { let altair_fork_epoch = Epoch::new(4); let altair_fork_slot = altair_fork_epoch.start_slot(E::slots_per_epoch()); let bellatrix_fork_epoch = Epoch::new(8); - let merge_fork_slot = bellatrix_fork_epoch.start_slot(E::slots_per_epoch()); + let bellatrix_fork_slot = bellatrix_fork_epoch.start_slot(E::slots_per_epoch()); let mut spec = E::default_spec(); spec.altair_fork_epoch = Some(altair_fork_epoch); @@ -130,41 +130,41 @@ async fn base_altair_merge_with_terminal_block_after_fork() { assert_eq!(altair_head.slot(), altair_fork_slot); /* - * Do the merge fork, without a terminal PoW block. + * Do the Bellatrix fork, without a terminal PoW block. */ - harness.extend_to_slot(merge_fork_slot).await; + harness.extend_to_slot(bellatrix_fork_slot).await; - let merge_head = &harness.chain.head_snapshot().beacon_block; - assert!(merge_head.as_merge().is_ok()); - assert_eq!(merge_head.slot(), merge_fork_slot); + let bellatrix_head = &harness.chain.head_snapshot().beacon_block; + assert!(bellatrix_head.as_bellatrix().is_ok()); + assert_eq!(bellatrix_head.slot(), bellatrix_fork_slot); assert!( - merge_head + bellatrix_head .message() .body() .execution_payload() .unwrap() .is_default_with_empty_roots(), - "Merge head is default payload" + "Bellatrix head is default payload" ); /* - * Next merge block shouldn't include an exec payload. + * Next Bellatrix block shouldn't include an exec payload. */ harness.extend_slots(1).await; - let one_after_merge_head = &harness.chain.head_snapshot().beacon_block; + let one_after_bellatrix_head = &harness.chain.head_snapshot().beacon_block; assert!( - one_after_merge_head + one_after_bellatrix_head .message() .body() .execution_payload() .unwrap() .is_default_with_empty_roots(), - "One after merge head is default payload" + "One after bellatrix head is default payload" ); - assert_eq!(one_after_merge_head.slot(), merge_fork_slot + 1); + assert_eq!(one_after_bellatrix_head.slot(), bellatrix_fork_slot + 1); /* * Trigger the terminal PoW block. @@ -188,20 +188,20 @@ async fn base_altair_merge_with_terminal_block_after_fork() { harness.extend_slots(1).await; - let two_after_merge_head = &harness.chain.head_snapshot().beacon_block; + let two_after_bellatrix_head = &harness.chain.head_snapshot().beacon_block; assert!( - two_after_merge_head + two_after_bellatrix_head .message() .body() .execution_payload() .unwrap() .is_default_with_empty_roots(), - "Two after merge head is default payload" + "Two after bellatrix head is default payload" ); - assert_eq!(two_after_merge_head.slot(), merge_fork_slot + 2); + assert_eq!(two_after_bellatrix_head.slot(), bellatrix_fork_slot + 2); /* - * Next merge block should include an exec payload. + * Next Bellatrix block should include an exec payload. */ for _ in 0..4 { harness.extend_slots(1).await; diff --git a/beacon_node/beacon_chain/tests/capella.rs b/beacon_node/beacon_chain/tests/capella.rs index dc40280f530..c8fd2637f0c 100644 --- a/beacon_node/beacon_chain/tests/capella.rs +++ b/beacon_node/beacon_chain/tests/capella.rs @@ -25,11 +25,11 @@ fn verify_execution_payload_chain(chain: &[FullPayload]) { } #[tokio::test] -async fn base_altair_merge_capella() { +async fn base_altair_bellatrix_capella() { let altair_fork_epoch = Epoch::new(4); let altair_fork_slot = altair_fork_epoch.start_slot(E::slots_per_epoch()); let bellatrix_fork_epoch = Epoch::new(8); - let merge_fork_slot = bellatrix_fork_epoch.start_slot(E::slots_per_epoch()); + let bellatrix_fork_slot = bellatrix_fork_epoch.start_slot(E::slots_per_epoch()); let capella_fork_epoch = Epoch::new(12); let capella_fork_slot = capella_fork_epoch.start_slot(E::slots_per_epoch()); @@ -61,39 +61,39 @@ async fn base_altair_merge_capella() { assert_eq!(altair_head.slot(), altair_fork_slot); /* - * Do the merge fork, without a terminal PoW block. + * Do the Bellatrix fork, without a terminal PoW block. */ - harness.extend_to_slot(merge_fork_slot).await; + harness.extend_to_slot(bellatrix_fork_slot).await; - let merge_head = &harness.chain.head_snapshot().beacon_block; - assert!(merge_head.as_merge().is_ok()); - assert_eq!(merge_head.slot(), merge_fork_slot); + let bellatrix_head = &harness.chain.head_snapshot().beacon_block; + assert!(bellatrix_head.as_bellatrix().is_ok()); + assert_eq!(bellatrix_head.slot(), bellatrix_fork_slot); assert!( - merge_head + bellatrix_head .message() .body() .execution_payload() .unwrap() .is_default_with_empty_roots(), - "Merge head is default payload" + "Bellatrix head is default payload" ); /* - * Next merge block shouldn't include an exec payload. + * Next Bellatrix block shouldn't include an exec payload. */ harness.extend_slots(1).await; - let one_after_merge_head = &harness.chain.head_snapshot().beacon_block; + let one_after_bellatrix_head = &harness.chain.head_snapshot().beacon_block; assert!( - one_after_merge_head + one_after_bellatrix_head .message() .body() .execution_payload() .unwrap() .is_default_with_empty_roots(), - "One after merge head is default payload" + "One after bellatrix head is default payload" ); - assert_eq!(one_after_merge_head.slot(), merge_fork_slot + 1); + assert_eq!(one_after_bellatrix_head.slot(), bellatrix_fork_slot + 1); /* * Trigger the terminal PoW block. @@ -114,23 +114,23 @@ async fn base_altair_merge_capella() { }); harness.extend_slots(1).await; - let two_after_merge_head = &harness.chain.head_snapshot().beacon_block; + let two_after_bellatrix_head = &harness.chain.head_snapshot().beacon_block; assert!( - two_after_merge_head + two_after_bellatrix_head .message() .body() .execution_payload() .unwrap() .is_default_with_empty_roots(), - "Two after merge head is default payload" + "Two after bellatrix head is default payload" ); - assert_eq!(two_after_merge_head.slot(), merge_fork_slot + 2); + assert_eq!(two_after_bellatrix_head.slot(), bellatrix_fork_slot + 2); /* - * Next merge block should include an exec payload. + * Next Bellatrix block should include an exec payload. */ let mut execution_payloads = vec![]; - for _ in (merge_fork_slot.as_u64() + 3)..capella_fork_slot.as_u64() { + for _ in (bellatrix_fork_slot.as_u64() + 3)..capella_fork_slot.as_u64() { harness.extend_slots(1).await; let block = &harness.chain.head_snapshot().beacon_block; let full_payload: FullPayload = diff --git a/beacon_node/beacon_chain/tests/main.rs b/beacon_node/beacon_chain/tests/main.rs index e0564e1510b..942ce816846 100644 --- a/beacon_node/beacon_chain/tests/main.rs +++ b/beacon_node/beacon_chain/tests/main.rs @@ -1,9 +1,9 @@ mod attestation_production; mod attestation_verification; +mod bellatrix; mod block_verification; mod capella; mod events; -mod merge; mod op_verification; mod payload_invalidation; mod rewards; diff --git a/beacon_node/beacon_chain/tests/validator_monitor.rs b/beacon_node/beacon_chain/tests/validator_monitor.rs index f595e5037e2..ea9ef73575f 100644 --- a/beacon_node/beacon_chain/tests/validator_monitor.rs +++ b/beacon_node/beacon_chain/tests/validator_monitor.rs @@ -211,7 +211,7 @@ async fn produces_missed_blocks() { // `validator_indexes[slot_in_epoch.as_usize()]` and add it below. let validator_index_to_monitor_altair = 2; // Same as above but for the merge upgrade - let validator_index_to_monitor_merge = 4; + let validator_index_to_monitor_bellatrix = 4; // Same as above but for the capella upgrade let validator_index_to_monitor_capella = 11; // Same as above but for the deneb upgrade @@ -224,7 +224,7 @@ async fn produces_missed_blocks() { vec![ validator_index_to_monitor, validator_index_to_monitor_altair, - validator_index_to_monitor_merge, + validator_index_to_monitor_bellatrix, validator_index_to_monitor_capella, validator_index_to_monitor_deneb, validator_index_to_monitor_electra, diff --git a/beacon_node/client/src/notifier.rs b/beacon_node/client/src/notifier.rs index 912babdae31..a6fd07789d8 100644 --- a/beacon_node/client/src/notifier.rs +++ b/beacon_node/client/src/notifier.rs @@ -1,9 +1,9 @@ use crate::metrics; use beacon_chain::{ + bellatrix_readiness::{BellatrixReadiness, GenesisExecutionPayloadStatus, MergeConfig}, capella_readiness::CapellaReadiness, deneb_readiness::DenebReadiness, electra_readiness::ElectraReadiness, - merge_readiness::{GenesisExecutionPayloadStatus, MergeConfig, MergeReadiness}, BeaconChain, BeaconChainTypes, ExecutionStatus, }; use lighthouse_network::{types::SyncState, NetworkGlobals}; @@ -64,7 +64,7 @@ pub fn spawn_notifier( "wait_time" => estimated_time_pretty(Some(next_slot.as_secs() as f64)), ); eth1_logging(&beacon_chain, &log); - merge_readiness_logging(Slot::new(0), &beacon_chain, &log).await; + bellatrix_readiness_logging(Slot::new(0), &beacon_chain, &log).await; capella_readiness_logging(Slot::new(0), &beacon_chain, &log).await; genesis_execution_payload_logging(&beacon_chain, &log).await; sleep(slot_duration).await; @@ -319,7 +319,7 @@ pub fn spawn_notifier( } eth1_logging(&beacon_chain, &log); - merge_readiness_logging(current_slot, &beacon_chain, &log).await; + bellatrix_readiness_logging(current_slot, &beacon_chain, &log).await; capella_readiness_logging(current_slot, &beacon_chain, &log).await; deneb_readiness_logging(current_slot, &beacon_chain, &log).await; electra_readiness_logging(current_slot, &beacon_chain, &log).await; @@ -334,7 +334,7 @@ pub fn spawn_notifier( /// Provides some helpful logging to users to indicate if their node is ready for the Bellatrix /// fork and subsequent merge transition. -async fn merge_readiness_logging( +async fn bellatrix_readiness_logging( current_slot: Slot, beacon_chain: &BeaconChain, log: &Logger, @@ -372,8 +372,8 @@ async fn merge_readiness_logging( return; } - match beacon_chain.check_merge_readiness(current_slot).await { - MergeReadiness::Ready { + match beacon_chain.check_bellatrix_readiness(current_slot).await { + BellatrixReadiness::Ready { config, current_difficulty, } => match config { @@ -384,7 +384,7 @@ async fn merge_readiness_logging( } => { info!( log, - "Ready for the merge"; + "Ready for Bellatrix"; "terminal_total_difficulty" => %ttd, "current_difficulty" => current_difficulty .map(|d| d.to_string()) @@ -398,7 +398,7 @@ async fn merge_readiness_logging( } => { info!( log, - "Ready for the merge"; + "Ready for Bellatrix"; "info" => "you are using override parameters, please ensure that you \ understand these parameters and their implications.", "terminal_block_hash" => ?terminal_block_hash, @@ -411,14 +411,14 @@ async fn merge_readiness_logging( "config" => ?other ), }, - readiness @ MergeReadiness::NotSynced => warn!( + readiness @ BellatrixReadiness::NotSynced => warn!( log, - "Not ready for merge"; + "Not ready Bellatrix"; "info" => %readiness, ), - readiness @ MergeReadiness::NoExecutionEndpoint => warn!( + readiness @ BellatrixReadiness::NoExecutionEndpoint => warn!( log, - "Not ready for merge"; + "Not ready for Bellatrix"; "info" => %readiness, ), } diff --git a/beacon_node/execution_layer/src/block_hash.rs b/beacon_node/execution_layer/src/block_hash.rs index 1f8c29f6b25..0d0cfaf352c 100644 --- a/beacon_node/execution_layer/src/block_hash.rs +++ b/beacon_node/execution_layer/src/block_hash.rs @@ -146,7 +146,7 @@ mod test { } #[test] - fn test_rlp_encode_merge_block() { + fn test_rlp_encode_bellatrix_block() { let header = ExecutionBlockHeader { parent_hash: Hash256::from_str("927ca537f06c783a3a2635b8805eef1c8c2124f7444ad4a3389898dd832f2dbe").unwrap(), ommers_hash: Hash256::from_str("1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347").unwrap(), diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index d422994c595..f3f059a4355 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -25,8 +25,8 @@ pub use types::{ Withdrawal, Withdrawals, }; use types::{ - ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadElectra, ExecutionPayloadMerge, - KzgProofs, + ExecutionPayloadBellatrix, ExecutionPayloadCapella, ExecutionPayloadDeneb, + ExecutionPayloadElectra, KzgProofs, }; use types::{Graffiti, GRAFFITI_BYTES_LEN}; @@ -36,8 +36,8 @@ pub mod json_structures; mod new_payload_request; pub use new_payload_request::{ - NewPayloadRequest, NewPayloadRequestCapella, NewPayloadRequestDeneb, NewPayloadRequestElectra, - NewPayloadRequestMerge, + NewPayloadRequest, NewPayloadRequestBellatrix, NewPayloadRequestCapella, + NewPayloadRequestDeneb, NewPayloadRequestElectra, }; pub const LATEST_TAG: &str = "latest"; @@ -155,7 +155,7 @@ pub struct ExecutionBlock { /// Representation of an execution block with enough detail to reconstruct a payload. #[superstruct( - variants(Merge, Capella, Deneb, Electra), + variants(Bellatrix, Capella, Deneb, Electra), variant_attributes( derive(Clone, Debug, PartialEq, Serialize, Deserialize,), serde(bound = "E: EthSpec", rename_all = "camelCase"), @@ -204,26 +204,28 @@ impl TryFrom> for ExecutionBlockWithTransactions fn try_from(payload: ExecutionPayload) -> Result { let json_payload = match payload { - ExecutionPayload::Merge(block) => Self::Merge(ExecutionBlockWithTransactionsMerge { - parent_hash: block.parent_hash, - fee_recipient: block.fee_recipient, - state_root: block.state_root, - receipts_root: block.receipts_root, - logs_bloom: block.logs_bloom, - prev_randao: block.prev_randao, - block_number: block.block_number, - gas_limit: block.gas_limit, - gas_used: block.gas_used, - timestamp: block.timestamp, - extra_data: block.extra_data, - base_fee_per_gas: block.base_fee_per_gas, - block_hash: block.block_hash, - transactions: block - .transactions - .iter() - .map(|tx| Transaction::decode(&Rlp::new(tx))) - .collect::, _>>()?, - }), + ExecutionPayload::Bellatrix(block) => { + Self::Bellatrix(ExecutionBlockWithTransactionsBellatrix { + parent_hash: block.parent_hash, + fee_recipient: block.fee_recipient, + state_root: block.state_root, + receipts_root: block.receipts_root, + logs_bloom: block.logs_bloom, + prev_randao: block.prev_randao, + block_number: block.block_number, + gas_limit: block.gas_limit, + gas_used: block.gas_used, + timestamp: block.timestamp, + extra_data: block.extra_data, + base_fee_per_gas: block.base_fee_per_gas, + block_hash: block.block_hash, + transactions: block + .transactions + .iter() + .map(|tx| Transaction::decode(&Rlp::new(tx))) + .collect::, _>>()?, + }) + } ExecutionPayload::Capella(block) => { Self::Capella(ExecutionBlockWithTransactionsCapella { parent_hash: block.parent_hash, @@ -423,7 +425,7 @@ pub struct ProposeBlindedBlockResponse { } #[superstruct( - variants(Merge, Capella, Deneb, Electra), + variants(Bellatrix, Capella, Deneb, Electra), variant_attributes(derive(Clone, Debug, PartialEq),), map_into(ExecutionPayload), map_ref_into(ExecutionPayloadRef), @@ -432,8 +434,11 @@ pub struct ProposeBlindedBlockResponse { )] #[derive(Clone, Debug, PartialEq)] pub struct GetPayloadResponse { - #[superstruct(only(Merge), partial_getter(rename = "execution_payload_merge"))] - pub execution_payload: ExecutionPayloadMerge, + #[superstruct( + only(Bellatrix), + partial_getter(rename = "execution_payload_bellatrix") + )] + pub execution_payload: ExecutionPayloadBellatrix, #[superstruct(only(Capella), partial_getter(rename = "execution_payload_capella"))] pub execution_payload: ExecutionPayloadCapella, #[superstruct(only(Deneb), partial_getter(rename = "execution_payload_deneb"))] @@ -482,8 +487,8 @@ impl From> { fn from(response: GetPayloadResponse) -> Self { match response { - GetPayloadResponse::Merge(inner) => ( - ExecutionPayload::Merge(inner.execution_payload), + GetPayloadResponse::Bellatrix(inner) => ( + ExecutionPayload::Bellatrix(inner.execution_payload), inner.block_value, None, ), @@ -529,14 +534,14 @@ impl ExecutionPayloadBodyV1 { header: ExecutionPayloadHeader, ) -> Result, String> { match header { - ExecutionPayloadHeader::Merge(header) => { + ExecutionPayloadHeader::Bellatrix(header) => { if self.withdrawals.is_some() { return Err(format!( "block {} is merge but payload body has withdrawals", header.block_hash )); } - Ok(ExecutionPayload::Merge(ExecutionPayloadMerge { + Ok(ExecutionPayload::Bellatrix(ExecutionPayloadBellatrix { parent_hash: header.parent_hash, fee_recipient: header.fee_recipient, state_root: header.state_root, diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index 405416e4f9d..93705a16925 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -737,7 +737,7 @@ impl HttpJsonRpc { ) -> Result>, Error> { let params = json!([block_hash, true]); Ok(Some(match fork { - ForkName::Merge => ExecutionBlockWithTransactions::Merge( + ForkName::Bellatrix => ExecutionBlockWithTransactions::Bellatrix( self.rpc_request( ETH_GET_BLOCK_BY_HASH, params, @@ -868,7 +868,7 @@ impl HttpJsonRpc { ) .await?; - Ok(GetPayloadResponse::Merge(GetPayloadResponseMerge { + Ok(GetPayloadResponse::Bellatrix(GetPayloadResponseBellatrix { execution_payload: payload_v1.into(), // Set the V1 payload values from the EE to be zero. This simulates // the pre-block-value functionality of always choosing the builder @@ -885,7 +885,7 @@ impl HttpJsonRpc { let params = json!([JsonPayloadIdRequest::from(payload_id)]); match fork_name { - ForkName::Merge => { + ForkName::Bellatrix => { let response: JsonGetPayloadResponseV1 = self .rpc_request( ENGINE_GET_PAYLOAD_V2, @@ -939,7 +939,7 @@ impl HttpJsonRpc { .await?; Ok(JsonGetPayloadResponse::V4(response).into()) } - ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => Err( + ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => Err( Error::UnsupportedForkVariant(format!("called get_payload_v3 with {}", fork_name)), ), } @@ -1180,7 +1180,7 @@ impl HttpJsonRpc { ) -> Result { let engine_capabilities = self.get_engine_capabilities(None).await?; match new_payload_request { - NewPayloadRequest::Merge(_) | NewPayloadRequest::Capella(_) => { + NewPayloadRequest::Bellatrix(_) | NewPayloadRequest::Capella(_) => { if engine_capabilities.new_payload_v2 { self.new_payload_v2(new_payload_request.into_execution_payload()) .await @@ -1218,7 +1218,7 @@ impl HttpJsonRpc { ) -> Result, Error> { let engine_capabilities = self.get_engine_capabilities(None).await?; match fork_name { - ForkName::Merge | ForkName::Capella => { + ForkName::Bellatrix | ForkName::Capella => { if engine_capabilities.get_payload_v2 { self.get_payload_v2(fork_name, payload_id).await } else if engine_capabilities.new_payload_v1 { @@ -1659,8 +1659,8 @@ mod test { .assert_request_equals( |client| async move { let _ = client - .new_payload_v1::(ExecutionPayload::Merge( - ExecutionPayloadMerge { + .new_payload_v1::(ExecutionPayload::Bellatrix( + ExecutionPayloadBellatrix { parent_hash: ExecutionBlockHash::repeat_byte(0), fee_recipient: Address::repeat_byte(1), state_root: Hash256::repeat_byte(1), @@ -1706,8 +1706,8 @@ mod test { Tester::new(false) .assert_auth_failure(|client| async move { client - .new_payload_v1::(ExecutionPayload::Merge( - ExecutionPayloadMerge { + .new_payload_v1::(ExecutionPayload::Bellatrix( + ExecutionPayloadBellatrix { parent_hash: ExecutionBlockHash::repeat_byte(0), fee_recipient: Address::repeat_byte(1), state_root: Hash256::repeat_byte(1), @@ -1917,7 +1917,7 @@ mod test { .unwrap() .into(); - let expected = ExecutionPayload::Merge(ExecutionPayloadMerge { + let expected = ExecutionPayload::Bellatrix(ExecutionPayloadBellatrix { parent_hash: ExecutionBlockHash::from_str("0x3b8fb240d288781d4aac94d3fd16809ee413bc99294a085798a589dae51ddd4a").unwrap(), fee_recipient: Address::from_str("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b").unwrap(), state_root: Hash256::from_str("0xca3149fa9e37db08d1cd49c9061db1002ef1cd58db2210f2115c8c989b2bdf45").unwrap(), @@ -1942,7 +1942,7 @@ mod test { // engine_newPayloadV1 REQUEST validation |client| async move { let _ = client - .new_payload_v1::(ExecutionPayload::Merge(ExecutionPayloadMerge{ + .new_payload_v1::(ExecutionPayload::Bellatrix(ExecutionPayloadBellatrix{ parent_hash: ExecutionBlockHash::from_str("0x3b8fb240d288781d4aac94d3fd16809ee413bc99294a085798a589dae51ddd4a").unwrap(), fee_recipient: Address::from_str("0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b").unwrap(), state_root: Hash256::from_str("0xca3149fa9e37db08d1cd49c9061db1002ef1cd58db2210f2115c8c989b2bdf45").unwrap(), @@ -1996,7 +1996,7 @@ mod test { })], |client| async move { let response = client - .new_payload_v1::(ExecutionPayload::Merge(ExecutionPayloadMerge::default())) + .new_payload_v1::(ExecutionPayload::Bellatrix(ExecutionPayloadBellatrix::default())) .await .unwrap(); diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index d784aa4cd9c..5c4d6ab1ac3 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -103,8 +103,8 @@ pub struct JsonExecutionPayload { pub excess_blob_gas: u64, } -impl From> for JsonExecutionPayloadV1 { - fn from(payload: ExecutionPayloadMerge) -> Self { +impl From> for JsonExecutionPayloadV1 { + fn from(payload: ExecutionPayloadBellatrix) -> Self { JsonExecutionPayloadV1 { parent_hash: payload.parent_hash, fee_recipient: payload.fee_recipient, @@ -210,7 +210,7 @@ impl From> for JsonExecutionPayloadV4 impl From> for JsonExecutionPayload { fn from(execution_payload: ExecutionPayload) -> Self { match execution_payload { - ExecutionPayload::Merge(payload) => JsonExecutionPayload::V1(payload.into()), + ExecutionPayload::Bellatrix(payload) => JsonExecutionPayload::V1(payload.into()), ExecutionPayload::Capella(payload) => JsonExecutionPayload::V2(payload.into()), ExecutionPayload::Deneb(payload) => JsonExecutionPayload::V3(payload.into()), ExecutionPayload::Electra(payload) => JsonExecutionPayload::V4(payload.into()), @@ -218,9 +218,9 @@ impl From> for JsonExecutionPayload { } } -impl From> for ExecutionPayloadMerge { +impl From> for ExecutionPayloadBellatrix { fn from(payload: JsonExecutionPayloadV1) -> Self { - ExecutionPayloadMerge { + ExecutionPayloadBellatrix { parent_hash: payload.parent_hash, fee_recipient: payload.fee_recipient, state_root: payload.state_root, @@ -326,7 +326,7 @@ impl From> for ExecutionPayloadElectra impl From> for ExecutionPayload { fn from(json_execution_payload: JsonExecutionPayload) -> Self { match json_execution_payload { - JsonExecutionPayload::V1(payload) => ExecutionPayload::Merge(payload.into()), + JsonExecutionPayload::V1(payload) => ExecutionPayload::Bellatrix(payload.into()), JsonExecutionPayload::V2(payload) => ExecutionPayload::Capella(payload.into()), JsonExecutionPayload::V3(payload) => ExecutionPayload::Deneb(payload.into()), JsonExecutionPayload::V4(payload) => ExecutionPayload::Electra(payload.into()), @@ -366,7 +366,7 @@ impl From> for GetPayloadResponse { fn from(json_get_payload_response: JsonGetPayloadResponse) -> Self { match json_get_payload_response { JsonGetPayloadResponse::V1(response) => { - GetPayloadResponse::Merge(GetPayloadResponseMerge { + GetPayloadResponse::Bellatrix(GetPayloadResponseBellatrix { execution_payload: response.execution_payload.into(), block_value: response.block_value, }) diff --git a/beacon_node/execution_layer/src/engine_api/new_payload_request.rs b/beacon_node/execution_layer/src/engine_api/new_payload_request.rs index 6b6df13b704..8d2e3d5ad06 100644 --- a/beacon_node/execution_layer/src/engine_api/new_payload_request.rs +++ b/beacon_node/execution_layer/src/engine_api/new_payload_request.rs @@ -8,11 +8,12 @@ use types::{ ExecutionPayloadRef, Hash256, VersionedHash, }; use types::{ - ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadElectra, ExecutionPayloadMerge, + ExecutionPayloadBellatrix, ExecutionPayloadCapella, ExecutionPayloadDeneb, + ExecutionPayloadElectra, }; #[superstruct( - variants(Merge, Capella, Deneb, Electra), + variants(Bellatrix, Capella, Deneb, Electra), variant_attributes(derive(Clone, Debug, PartialEq),), map_into(ExecutionPayload), map_ref_into(ExecutionPayloadRef), @@ -27,8 +28,11 @@ use types::{ )] #[derive(Clone, Debug, PartialEq)] pub struct NewPayloadRequest<'block, E: EthSpec> { - #[superstruct(only(Merge), partial_getter(rename = "execution_payload_merge"))] - pub execution_payload: &'block ExecutionPayloadMerge, + #[superstruct( + only(Bellatrix), + partial_getter(rename = "execution_payload_bellatrix") + )] + pub execution_payload: &'block ExecutionPayloadBellatrix, #[superstruct(only(Capella), partial_getter(rename = "execution_payload_capella"))] pub execution_payload: &'block ExecutionPayloadCapella, #[superstruct(only(Deneb), partial_getter(rename = "execution_payload_deneb"))] @@ -44,7 +48,7 @@ pub struct NewPayloadRequest<'block, E: EthSpec> { impl<'block, E: EthSpec> NewPayloadRequest<'block, E> { pub fn parent_hash(&self) -> ExecutionBlockHash { match self { - Self::Merge(payload) => payload.execution_payload.parent_hash, + Self::Bellatrix(payload) => payload.execution_payload.parent_hash, Self::Capella(payload) => payload.execution_payload.parent_hash, Self::Deneb(payload) => payload.execution_payload.parent_hash, Self::Electra(payload) => payload.execution_payload.parent_hash, @@ -53,7 +57,7 @@ impl<'block, E: EthSpec> NewPayloadRequest<'block, E> { pub fn block_hash(&self) -> ExecutionBlockHash { match self { - Self::Merge(payload) => payload.execution_payload.block_hash, + Self::Bellatrix(payload) => payload.execution_payload.block_hash, Self::Capella(payload) => payload.execution_payload.block_hash, Self::Deneb(payload) => payload.execution_payload.block_hash, Self::Electra(payload) => payload.execution_payload.block_hash, @@ -62,7 +66,7 @@ impl<'block, E: EthSpec> NewPayloadRequest<'block, E> { pub fn block_number(&self) -> u64 { match self { - Self::Merge(payload) => payload.execution_payload.block_number, + Self::Bellatrix(payload) => payload.execution_payload.block_number, Self::Capella(payload) => payload.execution_payload.block_number, Self::Deneb(payload) => payload.execution_payload.block_number, Self::Electra(payload) => payload.execution_payload.block_number, @@ -71,7 +75,7 @@ impl<'block, E: EthSpec> NewPayloadRequest<'block, E> { pub fn execution_payload_ref(&self) -> ExecutionPayloadRef<'block, E> { match self { - Self::Merge(request) => ExecutionPayloadRef::Merge(request.execution_payload), + Self::Bellatrix(request) => ExecutionPayloadRef::Bellatrix(request.execution_payload), Self::Capella(request) => ExecutionPayloadRef::Capella(request.execution_payload), Self::Deneb(request) => ExecutionPayloadRef::Deneb(request.execution_payload), Self::Electra(request) => ExecutionPayloadRef::Electra(request.execution_payload), @@ -80,7 +84,9 @@ impl<'block, E: EthSpec> NewPayloadRequest<'block, E> { pub fn into_execution_payload(self) -> ExecutionPayload { match self { - Self::Merge(request) => ExecutionPayload::Merge(request.execution_payload.clone()), + Self::Bellatrix(request) => { + ExecutionPayload::Bellatrix(request.execution_payload.clone()) + } Self::Capella(request) => ExecutionPayload::Capella(request.execution_payload.clone()), Self::Deneb(request) => ExecutionPayload::Deneb(request.execution_payload.clone()), Self::Electra(request) => ExecutionPayload::Electra(request.execution_payload.clone()), @@ -150,9 +156,11 @@ impl<'a, E: EthSpec> TryFrom> for NewPayloadRequest<'a, E> BeaconBlockRef::Base(_) | BeaconBlockRef::Altair(_) => { Err(Self::Error::IncorrectStateVariant) } - BeaconBlockRef::Merge(block_ref) => Ok(Self::Merge(NewPayloadRequestMerge { - execution_payload: &block_ref.body.execution_payload.execution_payload, - })), + BeaconBlockRef::Bellatrix(block_ref) => { + Ok(Self::Bellatrix(NewPayloadRequestBellatrix { + execution_payload: &block_ref.body.execution_payload.execution_payload, + })) + } BeaconBlockRef::Capella(block_ref) => Ok(Self::Capella(NewPayloadRequestCapella { execution_payload: &block_ref.body.execution_payload.execution_payload, })), @@ -185,9 +193,11 @@ impl<'a, E: EthSpec> TryFrom> for NewPayloadRequest<' fn try_from(payload: ExecutionPayloadRef<'a, E>) -> Result { match payload { - ExecutionPayloadRef::Merge(payload) => Ok(Self::Merge(NewPayloadRequestMerge { - execution_payload: payload, - })), + ExecutionPayloadRef::Bellatrix(payload) => { + Ok(Self::Bellatrix(NewPayloadRequestBellatrix { + execution_payload: payload, + })) + } ExecutionPayloadRef::Capella(payload) => Ok(Self::Capella(NewPayloadRequestCapella { execution_payload: payload, })), diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 60f450a39de..668a5ce84b2 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -50,8 +50,8 @@ use types::{ AbstractExecPayload, BlobsList, ExecutionPayloadDeneb, KzgProofs, SignedBlindedBeaconBlock, }; use types::{ - BeaconStateError, BlindedPayload, ChainSpec, Epoch, ExecPayload, ExecutionPayloadCapella, - ExecutionPayloadElectra, ExecutionPayloadMerge, FullPayload, ProposerPreparationData, + BeaconStateError, BlindedPayload, ChainSpec, Epoch, ExecPayload, ExecutionPayloadBellatrix, + ExecutionPayloadCapella, ExecutionPayloadElectra, FullPayload, ProposerPreparationData, PublicKeyBytes, Signature, Slot, }; @@ -98,8 +98,8 @@ impl TryFrom> for ProvenancedPayload) -> Result { let block_proposal_contents = match value { - BuilderBid::Merge(builder_bid) => BlockProposalContents::Payload { - payload: ExecutionPayloadHeader::Merge(builder_bid.header).into(), + BuilderBid::Bellatrix(builder_bid) => BlockProposalContents::Payload { + payload: ExecutionPayloadHeader::Bellatrix(builder_bid.header).into(), block_value: builder_bid.value, }, BuilderBid::Capella(builder_bid) => BlockProposalContents::Payload { @@ -1804,7 +1804,7 @@ impl ExecutionLayer { // Handle default payload body. if header.block_hash() == ExecutionBlockHash::zero() { let payload = match fork { - ForkName::Merge => ExecutionPayloadMerge::default().into(), + ForkName::Bellatrix => ExecutionPayloadBellatrix::default().into(), ForkName::Capella => ExecutionPayloadCapella::default().into(), ForkName::Deneb => ExecutionPayloadDeneb::default().into(), ForkName::Electra => ExecutionPayloadElectra::default().into(), @@ -1873,7 +1873,7 @@ impl ExecutionLayer { if hash == ExecutionBlockHash::zero() { return match fork { - ForkName::Merge => Ok(Some(ExecutionPayloadMerge::default().into())), + ForkName::Bellatrix => Ok(Some(ExecutionPayloadBellatrix::default().into())), ForkName::Capella => Ok(Some(ExecutionPayloadCapella::default().into())), ForkName::Deneb => Ok(Some(ExecutionPayloadDeneb::default().into())), ForkName::Electra => Ok(Some(ExecutionPayloadElectra::default().into())), @@ -1902,22 +1902,22 @@ impl ExecutionLayer { }; let payload = match block { - ExecutionBlockWithTransactions::Merge(merge_block) => { - ExecutionPayload::Merge(ExecutionPayloadMerge { - parent_hash: merge_block.parent_hash, - fee_recipient: merge_block.fee_recipient, - state_root: merge_block.state_root, - receipts_root: merge_block.receipts_root, - logs_bloom: merge_block.logs_bloom, - prev_randao: merge_block.prev_randao, - block_number: merge_block.block_number, - gas_limit: merge_block.gas_limit, - gas_used: merge_block.gas_used, - timestamp: merge_block.timestamp, - extra_data: merge_block.extra_data, - base_fee_per_gas: merge_block.base_fee_per_gas, - block_hash: merge_block.block_hash, - transactions: convert_transactions(merge_block.transactions)?, + ExecutionBlockWithTransactions::Bellatrix(bellatrix_block) => { + ExecutionPayload::Bellatrix(ExecutionPayloadBellatrix { + parent_hash: bellatrix_block.parent_hash, + fee_recipient: bellatrix_block.fee_recipient, + state_root: bellatrix_block.state_root, + receipts_root: bellatrix_block.receipts_root, + logs_bloom: bellatrix_block.logs_bloom, + prev_randao: bellatrix_block.prev_randao, + block_number: bellatrix_block.block_number, + gas_limit: bellatrix_block.gas_limit, + gas_used: bellatrix_block.gas_used, + timestamp: bellatrix_block.timestamp, + extra_data: bellatrix_block.extra_data, + base_fee_per_gas: bellatrix_block.base_fee_per_gas, + block_hash: bellatrix_block.block_hash, + transactions: convert_transactions(bellatrix_block.transactions)?, }) } ExecutionBlockWithTransactions::Capella(capella_block) => { diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index bac2304fa85..77e12bcef66 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -21,9 +21,9 @@ use std::sync::Arc; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; use types::{ - Blob, ChainSpec, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, - ExecutionPayloadDeneb, ExecutionPayloadElectra, ExecutionPayloadHeader, ExecutionPayloadMerge, - ForkName, Hash256, Transaction, Transactions, Uint256, + Blob, ChainSpec, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadBellatrix, + ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadElectra, + ExecutionPayloadHeader, ForkName, Hash256, Transaction, Transactions, Uint256, }; use super::DEFAULT_TERMINAL_BLOCK; @@ -92,7 +92,7 @@ impl Block { match self { Block::PoS(payload) => Some(payload.clone().try_into().unwrap()), Block::PoW(block) => Some( - ExecutionPayload::Merge(ExecutionPayloadMerge { + ExecutionPayload::Bellatrix(ExecutionPayloadBellatrix { block_hash: block.block_hash, ..Default::default() }) @@ -232,7 +232,7 @@ impl ExecutionBlockGenerator { Some(fork_time) if timestamp >= fork_time => ForkName::Deneb, _ => match self.shanghai_time { Some(fork_time) if timestamp >= fork_time => ForkName::Capella, - _ => ForkName::Merge, + _ => ForkName::Bellatrix, }, }, } @@ -569,7 +569,7 @@ impl ExecutionBlockGenerator { attributes: &PayloadAttributes, ) -> Result, String> { let mut execution_payload = match attributes { - PayloadAttributes::V1(pa) => ExecutionPayload::Merge(ExecutionPayloadMerge { + PayloadAttributes::V1(pa) => ExecutionPayload::Bellatrix(ExecutionPayloadBellatrix { parent_hash: head_block_hash, fee_recipient: pa.suggested_fee_recipient, receipts_root: Hash256::repeat_byte(42), @@ -586,7 +586,7 @@ impl ExecutionBlockGenerator { transactions: vec![].into(), }), PayloadAttributes::V2(pa) => match self.get_fork_at_timestamp(pa.timestamp) { - ForkName::Merge => ExecutionPayload::Merge(ExecutionPayloadMerge { + ForkName::Bellatrix => ExecutionPayload::Bellatrix(ExecutionPayloadBellatrix { parent_hash: head_block_hash, fee_recipient: pa.suggested_fee_recipient, receipts_root: Hash256::repeat_byte(42), @@ -665,7 +665,7 @@ impl ExecutionBlockGenerator { }; match execution_payload.fork_name() { - ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => {} + ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => {} ForkName::Deneb | ForkName::Electra => { // get random number between 0 and Max Blobs let mut rng = self.rng.lock(); @@ -782,14 +782,14 @@ pub fn generate_genesis_header( let empty_transactions_root = Transactions::::empty().tree_hash_root(); match genesis_fork { ForkName::Base | ForkName::Altair => None, - ForkName::Merge => { + ForkName::Bellatrix => { if post_transition_merge { - let mut header = ExecutionPayloadHeader::Merge(<_>::default()); + let mut header = ExecutionPayloadHeader::Bellatrix(<_>::default()); *header.block_hash_mut() = genesis_block_hash.unwrap_or_default(); *header.transactions_root_mut() = empty_transactions_root; Some(header) } else { - Some(ExecutionPayloadHeader::::Merge(<_>::default())) + Some(ExecutionPayloadHeader::::Bellatrix(<_>::default())) } } ForkName::Capella => { diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index c16bd43c82c..1dc8f0ab83e 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -135,7 +135,7 @@ pub async fn handle_rpc( .get_fork_at_timestamp(*request.timestamp()); // validate method called correctly according to fork time match fork { - ForkName::Merge => { + ForkName::Bellatrix => { if matches!(request, JsonExecutionPayload::V2(_)) { return Err(( format!( @@ -395,7 +395,7 @@ pub async fn handle_rpc( .read() .get_fork_at_timestamp(*pa.timestamp()) { - ForkName::Merge => { + ForkName::Bellatrix => { get_param::>(params, 1) .map(|opt| opt.map(JsonPayloadAttributes::V1)) .transpose() @@ -427,7 +427,7 @@ pub async fn handle_rpc( .read() .get_fork_at_timestamp(*pa.timestamp()) { - ForkName::Merge => { + ForkName::Bellatrix => { if matches!(pa, JsonPayloadAttributes::V2(_)) { return Err(( format!( diff --git a/beacon_node/execution_layer/src/test_utils/mock_builder.rs b/beacon_node/execution_layer/src/test_utils/mock_builder.rs index 756e0b793f8..c9ae1e60cdc 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_builder.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_builder.rs @@ -15,7 +15,7 @@ use task_executor::TaskExecutor; use tempfile::NamedTempFile; use tree_hash::TreeHash; use types::builder_bid::{ - BuilderBid, BuilderBidCapella, BuilderBidDeneb, BuilderBidElectra, BuilderBidMerge, + BuilderBid, BuilderBidBellatrix, BuilderBidCapella, BuilderBidDeneb, BuilderBidElectra, SignedBuilderBid, }; use types::{ @@ -77,7 +77,7 @@ pub trait BidStuff { impl BidStuff for BuilderBid { fn set_fee_recipient(&mut self, fee_recipient: Address) { match self.to_mut().header_mut() { - ExecutionPayloadHeaderRefMut::Merge(header) => { + ExecutionPayloadHeaderRefMut::Bellatrix(header) => { header.fee_recipient = fee_recipient; } ExecutionPayloadHeaderRefMut::Capella(header) => { @@ -94,7 +94,7 @@ impl BidStuff for BuilderBid { fn set_gas_limit(&mut self, gas_limit: u64) { match self.to_mut().header_mut() { - ExecutionPayloadHeaderRefMut::Merge(header) => { + ExecutionPayloadHeaderRefMut::Bellatrix(header) => { header.gas_limit = gas_limit; } ExecutionPayloadHeaderRefMut::Capella(header) => { @@ -115,7 +115,7 @@ impl BidStuff for BuilderBid { fn set_parent_hash(&mut self, parent_hash: Hash256) { match self.to_mut().header_mut() { - ExecutionPayloadHeaderRefMut::Merge(header) => { + ExecutionPayloadHeaderRefMut::Bellatrix(header) => { header.parent_hash = ExecutionBlockHash::from_root(parent_hash); } ExecutionPayloadHeaderRefMut::Capella(header) => { @@ -132,7 +132,7 @@ impl BidStuff for BuilderBid { fn set_prev_randao(&mut self, prev_randao: Hash256) { match self.to_mut().header_mut() { - ExecutionPayloadHeaderRefMut::Merge(header) => { + ExecutionPayloadHeaderRefMut::Bellatrix(header) => { header.prev_randao = prev_randao; } ExecutionPayloadHeaderRefMut::Capella(header) => { @@ -149,7 +149,7 @@ impl BidStuff for BuilderBid { fn set_block_number(&mut self, block_number: u64) { match self.to_mut().header_mut() { - ExecutionPayloadHeaderRefMut::Merge(header) => { + ExecutionPayloadHeaderRefMut::Bellatrix(header) => { header.block_number = block_number; } ExecutionPayloadHeaderRefMut::Capella(header) => { @@ -166,7 +166,7 @@ impl BidStuff for BuilderBid { fn set_timestamp(&mut self, timestamp: u64) { match self.to_mut().header_mut() { - ExecutionPayloadHeaderRefMut::Merge(header) => { + ExecutionPayloadHeaderRefMut::Bellatrix(header) => { header.timestamp = timestamp; } ExecutionPayloadHeaderRefMut::Capella(header) => { @@ -183,7 +183,7 @@ impl BidStuff for BuilderBid { fn set_withdrawals_root(&mut self, withdrawals_root: Hash256) { match self.to_mut().header_mut() { - ExecutionPayloadHeaderRefMut::Merge(_) => { + ExecutionPayloadHeaderRefMut::Bellatrix(_) => { panic!("no withdrawals before capella") } ExecutionPayloadHeaderRefMut::Capella(header) => { @@ -336,7 +336,7 @@ pub fn serve( SignedBlindedBeaconBlock::Base(_) | types::SignedBeaconBlock::Altair(_) => { return Err(reject("invalid fork")); } - SignedBlindedBeaconBlock::Merge(block) => { + SignedBlindedBeaconBlock::Bellatrix(block) => { block.message.body.execution_payload.tree_hash_root() } SignedBlindedBeaconBlock::Capella(block) => { @@ -480,7 +480,7 @@ pub fn serve( .get_randao_mix(head_state.current_epoch()) .map_err(|_| reject("couldn't get prev randao"))?; let expected_withdrawals = match fork { - ForkName::Base | ForkName::Altair | ForkName::Merge => None, + ForkName::Base | ForkName::Altair | ForkName::Bellatrix => None, ForkName::Capella | ForkName::Deneb | ForkName::Electra => Some( builder .beacon_client @@ -496,7 +496,7 @@ pub fn serve( // first to avoid polluting the execution block generator with invalid payload attributes // NOTE: this was part of an effort to add payload attribute uniqueness checks, // which was abandoned because it broke too many tests in subtle ways. - ForkName::Merge | ForkName::Capella => PayloadAttributes::new( + ForkName::Bellatrix | ForkName::Capella => PayloadAttributes::new( timestamp, *prev_randao, fee_recipient, @@ -577,9 +577,9 @@ pub fn serve( value: Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI), pubkey: builder.builder_sk.public_key().compress(), }), - ForkName::Merge => BuilderBid::Merge(BuilderBidMerge { + ForkName::Bellatrix => BuilderBid::Bellatrix(BuilderBidBellatrix { header: payload - .as_merge() + .as_bellatrix() .map_err(|_| reject("incorrect payload variant"))? .into(), value: Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI), @@ -627,9 +627,9 @@ pub fn serve( value: Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI), pubkey: builder.builder_sk.public_key().compress(), }), - ForkName::Merge => BuilderBid::Merge(BuilderBidMerge { + ForkName::Bellatrix => BuilderBid::Bellatrix(BuilderBidBellatrix { header: payload - .as_merge() + .as_bellatrix() .map_err(|_| reject("incorrect payload variant"))? .into(), value: Uint256::from(DEFAULT_BUILDER_PAYLOAD_VALUE_WEI), diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index 6717bbc2ab3..da9b2817f69 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -138,7 +138,7 @@ impl MockExecutionLayer { &payload_attributes, forkchoice_update_params, builder_params, - ForkName::Merge, + ForkName::Bellatrix, &self.spec, None, BlockProductionVersion::FullV2, @@ -178,7 +178,7 @@ impl MockExecutionLayer { &payload_attributes, forkchoice_update_params, builder_params, - ForkName::Merge, + ForkName::Bellatrix, &self.spec, None, BlockProductionVersion::BlindedV2, diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index 7e3778b3fbb..05a6735b327 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -12,7 +12,7 @@ pub fn build_block_contents( Ok(ProduceBlockV3Response::Blinded(block.block)) } BeaconBlockResponseWrapper::Full(block) => match fork_name { - ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => Ok( + ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => Ok( ProduceBlockV3Response::Full(FullBlockContents::Block(block.block)), ), ForkName::Deneb | ForkName::Electra => { diff --git a/beacon_node/http_api/src/builder_states.rs b/beacon_node/http_api/src/builder_states.rs index 90203f2d60c..a540113ab43 100644 --- a/beacon_node/http_api/src/builder_states.rs +++ b/beacon_node/http_api/src/builder_states.rs @@ -53,7 +53,7 @@ fn get_next_withdrawals_sanity_checks( } let fork = chain.spec.fork_name_at_slot::(proposal_slot); - if let ForkName::Base | ForkName::Altair | ForkName::Merge = fork { + if let ForkName::Base | ForkName::Altair | ForkName::Bellatrix = fork { return Err(warp_utils::reject::custom_bad_request( "the specified state is a pre-capella state.".to_string(), )); diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 46d3ad7569b..5f4620589eb 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -4296,7 +4296,7 @@ pub fn serve( |task_spawner: TaskSpawner, chain: Arc>| { task_spawner.spawn_async_with_rejection(Priority::P1, async move { let current_slot = chain.slot_clock.now_or_genesis().unwrap_or(Slot::new(0)); - let merge_readiness = chain.check_merge_readiness(current_slot).await; + let merge_readiness = chain.check_bellatrix_readiness(current_slot).await; Ok::<_, warp::reject::Rejection>( warp::reply::json(&api_types::GenericResponse::from(merge_readiness)) .into_response(), diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 52e77753ce4..0d176e6a53a 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -20,8 +20,8 @@ use tokio::sync::mpsc::UnboundedSender; use tree_hash::TreeHash; use types::{ AbstractExecPayload, BeaconBlockRef, BlobSidecarList, EthSpec, ExecPayload, ExecutionBlockHash, - ForkName, FullPayload, FullPayloadMerge, Hash256, SignedBeaconBlock, SignedBlindedBeaconBlock, - VariableList, + ForkName, FullPayload, FullPayloadBellatrix, Hash256, SignedBeaconBlock, + SignedBlindedBeaconBlock, VariableList, }; use warp::http::StatusCode; use warp::{reply::Response, Rejection, Reply}; @@ -80,7 +80,7 @@ pub async fn publish_block { crate::publish_pubsub_message(&sender, PubsubMessage::BeaconBlock(block)) .map_err(|_| BlockError::BeaconChainError(BeaconChainError::UnableToPublish))?; @@ -331,8 +331,8 @@ pub async fn reconstruct_block( let fork_name = chain .spec .fork_name_at_epoch(block.slot().epoch(T::EthSpec::slots_per_epoch())); - if fork_name == ForkName::Merge { - let payload: FullPayload = FullPayloadMerge::default().into(); + if fork_name == ForkName::Bellatrix { + let payload: FullPayload = FullPayloadBellatrix::default().into(); ProvenancedPayload::Local(FullPayloadContents::Payload(payload.into())) } else { Err(warp_utils::reject::custom_server_error( diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 03f530db4d1..91c5b62d0b2 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -438,7 +438,7 @@ pub fn gossipsub_config( let topic_bytes = message.topic.as_str().as_bytes(); match fork_context.current_fork() { ForkName::Altair - | ForkName::Merge + | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb | ForkName::Electra => { @@ -461,7 +461,7 @@ pub fn gossipsub_config( } } let message_domain_valid_snappy = gossipsub_config_params.message_domain_valid_snappy; - let is_merge_enabled = fork_context.fork_exists(ForkName::Merge); + let is_bellatrix_enabled = fork_context.fork_exists(ForkName::Bellatrix); let gossip_message_id = move |message: &gossipsub::Message| { gossipsub::MessageId::from( &Sha256::digest( @@ -481,7 +481,7 @@ pub fn gossipsub_config( gossipsub::ConfigBuilder::default() .max_transmit_size(gossip_max_size( - is_merge_enabled, + is_bellatrix_enabled, gossipsub_config_params.gossip_max_size, )) .heartbeat_interval(load.heartbeat_interval) diff --git a/beacon_node/lighthouse_network/src/rpc/codec/base.rs b/beacon_node/lighthouse_network/src/rpc/codec/base.rs index 287f0a3f5fd..42a31d3480a 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec/base.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec/base.rs @@ -184,13 +184,13 @@ mod tests { fn fork_context(fork_name: ForkName) -> ForkContext { let mut chain_spec = Spec::default_spec(); let altair_fork_epoch = Epoch::new(1); - let merge_fork_epoch = Epoch::new(2); + let bellatrix_fork_epoch = Epoch::new(2); let capella_fork_epoch = Epoch::new(3); let deneb_fork_epoch = Epoch::new(4); let electra_fork_epoch = Epoch::new(5); chain_spec.altair_fork_epoch = Some(altair_fork_epoch); - chain_spec.bellatrix_fork_epoch = Some(merge_fork_epoch); + chain_spec.bellatrix_fork_epoch = Some(bellatrix_fork_epoch); chain_spec.capella_fork_epoch = Some(capella_fork_epoch); chain_spec.deneb_fork_epoch = Some(deneb_fork_epoch); chain_spec.electra_fork_epoch = Some(electra_fork_epoch); @@ -198,7 +198,7 @@ mod tests { let current_slot = match fork_name { ForkName::Base => Slot::new(0), ForkName::Altair => altair_fork_epoch.start_slot(Spec::slots_per_epoch()), - ForkName::Merge => merge_fork_epoch.start_slot(Spec::slots_per_epoch()), + ForkName::Bellatrix => bellatrix_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Capella => capella_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Deneb => deneb_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Electra => electra_fork_epoch.start_slot(Spec::slots_per_epoch()), diff --git a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs index 4cbb6582583..482d1d96b4a 100644 --- a/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs +++ b/beacon_node/lighthouse_network/src/rpc/codec/ssz_snappy.rs @@ -18,8 +18,8 @@ use tokio_util::codec::{Decoder, Encoder}; use types::{ BlobSidecar, ChainSpec, EthSpec, ForkContext, ForkName, Hash256, LightClientBootstrap, LightClientFinalityUpdate, LightClientOptimisticUpdate, RuntimeVariableList, SignedBeaconBlock, - SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockCapella, - SignedBeaconBlockDeneb, SignedBeaconBlockElectra, SignedBeaconBlockMerge, + SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockBellatrix, + SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockElectra, }; use unsigned_varint::codec::Uvi; @@ -403,8 +403,8 @@ fn context_bytes( SignedBeaconBlock::Capella { .. } => { fork_context.to_context_bytes(ForkName::Capella) } - SignedBeaconBlock::Merge { .. } => { - fork_context.to_context_bytes(ForkName::Merge) + SignedBeaconBlock::Bellatrix { .. } => { + fork_context.to_context_bytes(ForkName::Bellatrix) } SignedBeaconBlock::Altair { .. } => { fork_context.to_context_bytes(ForkName::Altair) @@ -658,8 +658,10 @@ fn handle_rpc_response( Some(ForkName::Base) => Ok(Some(RPCResponse::BlocksByRange(Arc::new( SignedBeaconBlock::Base(SignedBeaconBlockBase::from_ssz_bytes(decoded_buffer)?), )))), - Some(ForkName::Merge) => Ok(Some(RPCResponse::BlocksByRange(Arc::new( - SignedBeaconBlock::Merge(SignedBeaconBlockMerge::from_ssz_bytes(decoded_buffer)?), + Some(ForkName::Bellatrix) => Ok(Some(RPCResponse::BlocksByRange(Arc::new( + SignedBeaconBlock::Bellatrix(SignedBeaconBlockBellatrix::from_ssz_bytes( + decoded_buffer, + )?), )))), Some(ForkName::Capella) => Ok(Some(RPCResponse::BlocksByRange(Arc::new( SignedBeaconBlock::Capella(SignedBeaconBlockCapella::from_ssz_bytes( @@ -689,8 +691,10 @@ fn handle_rpc_response( Some(ForkName::Base) => Ok(Some(RPCResponse::BlocksByRoot(Arc::new( SignedBeaconBlock::Base(SignedBeaconBlockBase::from_ssz_bytes(decoded_buffer)?), )))), - Some(ForkName::Merge) => Ok(Some(RPCResponse::BlocksByRoot(Arc::new( - SignedBeaconBlock::Merge(SignedBeaconBlockMerge::from_ssz_bytes(decoded_buffer)?), + Some(ForkName::Bellatrix) => Ok(Some(RPCResponse::BlocksByRoot(Arc::new( + SignedBeaconBlock::Bellatrix(SignedBeaconBlockBellatrix::from_ssz_bytes( + decoded_buffer, + )?), )))), Some(ForkName::Capella) => Ok(Some(RPCResponse::BlocksByRoot(Arc::new( SignedBeaconBlock::Capella(SignedBeaconBlockCapella::from_ssz_bytes( @@ -743,7 +747,7 @@ mod tests { use crate::types::{EnrAttestationBitfield, EnrSyncCommitteeBitfield}; use types::{ blob_sidecar::BlobIdentifier, BeaconBlock, BeaconBlockAltair, BeaconBlockBase, - BeaconBlockMerge, EmptyBlock, Epoch, FullPayload, Signature, Slot, + BeaconBlockBellatrix, EmptyBlock, Epoch, FullPayload, Signature, Slot, }; type Spec = types::MainnetEthSpec; @@ -751,13 +755,13 @@ mod tests { fn fork_context(fork_name: ForkName) -> ForkContext { let mut chain_spec = Spec::default_spec(); let altair_fork_epoch = Epoch::new(1); - let merge_fork_epoch = Epoch::new(2); + let bellatrix_fork_epoch = Epoch::new(2); let capella_fork_epoch = Epoch::new(3); let deneb_fork_epoch = Epoch::new(4); let electra_fork_epoch = Epoch::new(5); chain_spec.altair_fork_epoch = Some(altair_fork_epoch); - chain_spec.bellatrix_fork_epoch = Some(merge_fork_epoch); + chain_spec.bellatrix_fork_epoch = Some(bellatrix_fork_epoch); chain_spec.capella_fork_epoch = Some(capella_fork_epoch); chain_spec.deneb_fork_epoch = Some(deneb_fork_epoch); chain_spec.electra_fork_epoch = Some(electra_fork_epoch); @@ -765,7 +769,7 @@ mod tests { let current_slot = match fork_name { ForkName::Base => Slot::new(0), ForkName::Altair => altair_fork_epoch.start_slot(Spec::slots_per_epoch()), - ForkName::Merge => merge_fork_epoch.start_slot(Spec::slots_per_epoch()), + ForkName::Bellatrix => bellatrix_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Capella => capella_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Deneb => deneb_fork_epoch.start_slot(Spec::slots_per_epoch()), ForkName::Electra => electra_fork_epoch.start_slot(Spec::slots_per_epoch()), @@ -790,32 +794,38 @@ mod tests { Arc::new(BlobSidecar::empty()) } - /// Merge block with length < max_rpc_size. - fn merge_block_small(fork_context: &ForkContext, spec: &ChainSpec) -> SignedBeaconBlock { - let mut block: BeaconBlockMerge<_, FullPayload> = - BeaconBlockMerge::empty(&Spec::default_spec()); + /// Bellatrix block with length < max_rpc_size. + fn bellatrix_block_small( + fork_context: &ForkContext, + spec: &ChainSpec, + ) -> SignedBeaconBlock { + let mut block: BeaconBlockBellatrix<_, FullPayload> = + BeaconBlockBellatrix::empty(&Spec::default_spec()); let tx = VariableList::from(vec![0; 1024]); let txs = VariableList::from(std::iter::repeat(tx).take(5000).collect::>()); block.body.execution_payload.execution_payload.transactions = txs; - let block = BeaconBlock::Merge(block); + let block = BeaconBlock::Bellatrix(block); assert!(block.ssz_bytes_len() <= max_rpc_size(fork_context, spec.max_chunk_size as usize)); SignedBeaconBlock::from_block(block, Signature::empty()) } - /// Merge block with length > MAX_RPC_SIZE. - /// The max limit for a merge block is in the order of ~16GiB which wouldn't fit in memory. - /// Hence, we generate a merge block just greater than `MAX_RPC_SIZE` to test rejection on the rpc layer. - fn merge_block_large(fork_context: &ForkContext, spec: &ChainSpec) -> SignedBeaconBlock { - let mut block: BeaconBlockMerge<_, FullPayload> = - BeaconBlockMerge::empty(&Spec::default_spec()); + /// Bellatrix block with length > MAX_RPC_SIZE. + /// The max limit for a Bellatrix block is in the order of ~16GiB which wouldn't fit in memory. + /// Hence, we generate a Bellatrix block just greater than `MAX_RPC_SIZE` to test rejection on the rpc layer. + fn bellatrix_block_large( + fork_context: &ForkContext, + spec: &ChainSpec, + ) -> SignedBeaconBlock { + let mut block: BeaconBlockBellatrix<_, FullPayload> = + BeaconBlockBellatrix::empty(&Spec::default_spec()); let tx = VariableList::from(vec![0; 1024]); let txs = VariableList::from(std::iter::repeat(tx).take(100000).collect::>()); block.body.execution_payload.execution_payload.transactions = txs; - let block = BeaconBlock::Merge(block); + let block = BeaconBlock::Bellatrix(block); assert!(block.ssz_bytes_len() > max_rpc_size(fork_context, spec.max_chunk_size as usize)); SignedBeaconBlock::from_block(block, Signature::empty()) } @@ -1172,25 +1182,27 @@ mod tests { Ok(Some(RPCResponse::BlocksByRange(Arc::new(altair_block())))) ); - let merge_block_small = merge_block_small(&fork_context(ForkName::Merge), &chain_spec); - let merge_block_large = merge_block_large(&fork_context(ForkName::Merge), &chain_spec); + let bellatrix_block_small = + bellatrix_block_small(&fork_context(ForkName::Bellatrix), &chain_spec); + let bellatrix_block_large = + bellatrix_block_large(&fork_context(ForkName::Bellatrix), &chain_spec); assert_eq!( encode_then_decode_response( SupportedProtocol::BlocksByRangeV2, RPCCodedResponse::Success(RPCResponse::BlocksByRange(Arc::new( - merge_block_small.clone() + bellatrix_block_small.clone() ))), - ForkName::Merge, + ForkName::Bellatrix, &chain_spec, ), Ok(Some(RPCResponse::BlocksByRange(Arc::new( - merge_block_small.clone() + bellatrix_block_small.clone() )))) ); let mut encoded = - encode_without_length_checks(merge_block_large.as_ssz_bytes(), ForkName::Merge) + encode_without_length_checks(bellatrix_block_large.as_ssz_bytes(), ForkName::Bellatrix) .unwrap(); assert!( @@ -1198,7 +1210,7 @@ mod tests { decode_response( SupportedProtocol::BlocksByRangeV2, &mut encoded, - ForkName::Merge, + ForkName::Bellatrix, &chain_spec, ) .unwrap_err(), @@ -1248,16 +1260,18 @@ mod tests { encode_then_decode_response( SupportedProtocol::BlocksByRootV2, RPCCodedResponse::Success(RPCResponse::BlocksByRoot(Arc::new( - merge_block_small.clone() + bellatrix_block_small.clone() ))), - ForkName::Merge, + ForkName::Bellatrix, &chain_spec, ), - Ok(Some(RPCResponse::BlocksByRoot(Arc::new(merge_block_small)))) + Ok(Some(RPCResponse::BlocksByRoot(Arc::new( + bellatrix_block_small + )))) ); let mut encoded = - encode_without_length_checks(merge_block_large.as_ssz_bytes(), ForkName::Merge) + encode_without_length_checks(bellatrix_block_large.as_ssz_bytes(), ForkName::Bellatrix) .unwrap(); assert!( @@ -1265,7 +1279,7 @@ mod tests { decode_response( SupportedProtocol::BlocksByRootV2, &mut encoded, - ForkName::Merge, + ForkName::Bellatrix, &chain_spec, ) .unwrap_err(), diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index f65586087c2..12a7f09338e 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -17,8 +17,8 @@ use tokio_util::{ compat::{Compat, FuturesAsyncReadCompatExt}, }; use types::{ - BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockCapella, BeaconBlockElectra, - BeaconBlockMerge, BlobSidecar, ChainSpec, EmptyBlock, EthSpec, ForkContext, ForkName, + BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockBellatrix, BeaconBlockCapella, + BeaconBlockElectra, BlobSidecar, ChainSpec, EmptyBlock, EthSpec, ForkContext, ForkName, LightClientBootstrap, LightClientBootstrapAltair, LightClientFinalityUpdate, LightClientFinalityUpdateAltair, LightClientOptimisticUpdate, LightClientOptimisticUpdateAltair, MainnetEthSpec, Signature, SignedBeaconBlock, @@ -53,8 +53,8 @@ lazy_static! { .as_ssz_bytes() .len(); - pub static ref SIGNED_BEACON_BLOCK_MERGE_MIN: usize = SignedBeaconBlock::::from_block( - BeaconBlock::Merge(BeaconBlockMerge::::empty(&MainnetEthSpec::default_spec())), + pub static ref SIGNED_BEACON_BLOCK_BELLATRIX_MIN: usize = SignedBeaconBlock::::from_block( + BeaconBlock::Bellatrix(BeaconBlockBellatrix::::empty(&MainnetEthSpec::default_spec())), Signature::empty(), ) .as_ssz_bytes() @@ -74,14 +74,14 @@ lazy_static! { .as_ssz_bytes() .len(); - /// The `BeaconBlockMerge` block has an `ExecutionPayload` field which has a max size ~16 GiB for future proofing. + /// The `BeaconBlockBellatrix` block has an `ExecutionPayload` field which has a max size ~16 GiB for future proofing. /// We calculate the value from its fields instead of constructing the block and checking the length. /// Note: This is only the theoretical upper bound. We further bound the max size we receive over the network /// with `max_chunk_size`. - pub static ref SIGNED_BEACON_BLOCK_MERGE_MAX: usize = + pub static ref SIGNED_BEACON_BLOCK_BELLATRIX_MAX: usize = // Size of a full altair block *SIGNED_BEACON_BLOCK_ALTAIR_MAX - + types::ExecutionPayload::::max_execution_payload_merge_size() // adding max size of execution payload (~16gb) + + types::ExecutionPayload::::max_execution_payload_bellatrix_size() // adding max size of execution payload (~16gb) + ssz::BYTES_PER_LENGTH_OFFSET; // Adding the additional ssz offset for the `ExecutionPayload` field pub static ref SIGNED_BEACON_BLOCK_CAPELLA_MAX: usize = *SIGNED_BEACON_BLOCK_CAPELLA_MAX_WITHOUT_PAYLOAD @@ -134,7 +134,7 @@ const REQUEST_TIMEOUT: u64 = 15; pub fn max_rpc_size(fork_context: &ForkContext, max_chunk_size: usize) -> usize { match fork_context.current_fork() { ForkName::Altair | ForkName::Base => max_chunk_size / 10, - ForkName::Merge => max_chunk_size, + ForkName::Bellatrix => max_chunk_size, ForkName::Capella => max_chunk_size, ForkName::Deneb => max_chunk_size, ForkName::Electra => max_chunk_size, @@ -154,20 +154,20 @@ pub fn rpc_block_limits_by_fork(current_fork: ForkName) -> RpcLimits { *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair blocks *SIGNED_BEACON_BLOCK_ALTAIR_MAX, // Altair block is larger than base blocks ), - ForkName::Merge => RpcLimits::new( - *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and merge blocks - *SIGNED_BEACON_BLOCK_MERGE_MAX, // Merge block is larger than base and altair blocks + ForkName::Bellatrix => RpcLimits::new( + *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and bellatrix blocks + *SIGNED_BEACON_BLOCK_BELLATRIX_MAX, // Bellatrix block is larger than base and altair blocks ), ForkName::Capella => RpcLimits::new( - *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and merge blocks + *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and bellatrix blocks *SIGNED_BEACON_BLOCK_CAPELLA_MAX, // Capella block is larger than base, altair and merge blocks ), ForkName::Deneb => RpcLimits::new( - *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and merge blocks + *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and bellatrix blocks *SIGNED_BEACON_BLOCK_DENEB_MAX, // Deneb block is larger than all prior fork blocks ), ForkName::Electra => RpcLimits::new( - *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and merge blocks + *SIGNED_BEACON_BLOCK_BASE_MIN, // Base block is smaller than altair and bellatrix blocks *SIGNED_BEACON_BLOCK_ELECTRA_MAX, // Electra block is larger than Deneb block ), } @@ -178,7 +178,9 @@ fn rpc_light_client_finality_update_limits_by_fork(current_fork: ForkName) -> Rp match ¤t_fork { ForkName::Base => RpcLimits::new(0, 0), - ForkName::Altair | ForkName::Merge => RpcLimits::new(altair_fixed_len, altair_fixed_len), + ForkName::Altair | ForkName::Bellatrix => { + RpcLimits::new(altair_fixed_len, altair_fixed_len) + } ForkName::Capella => { RpcLimits::new(altair_fixed_len, *LIGHT_CLIENT_FINALITY_UPDATE_CAPELLA_MAX) } @@ -196,7 +198,9 @@ fn rpc_light_client_optimistic_update_limits_by_fork(current_fork: ForkName) -> match ¤t_fork { ForkName::Base => RpcLimits::new(0, 0), - ForkName::Altair | ForkName::Merge => RpcLimits::new(altair_fixed_len, altair_fixed_len), + ForkName::Altair | ForkName::Bellatrix => { + RpcLimits::new(altair_fixed_len, altair_fixed_len) + } ForkName::Capella => RpcLimits::new( altair_fixed_len, *LIGHT_CLIENT_OPTIMISTIC_UPDATE_CAPELLA_MAX, @@ -216,7 +220,9 @@ fn rpc_light_client_bootstrap_limits_by_fork(current_fork: ForkName) -> RpcLimit match ¤t_fork { ForkName::Base => RpcLimits::new(0, 0), - ForkName::Altair | ForkName::Merge => RpcLimits::new(altair_fixed_len, altair_fixed_len), + ForkName::Altair | ForkName::Bellatrix => { + RpcLimits::new(altair_fixed_len, altair_fixed_len) + } ForkName::Capella => RpcLimits::new(altair_fixed_len, *LIGHT_CLIENT_BOOTSTRAP_CAPELLA_MAX), ForkName::Deneb => RpcLimits::new(altair_fixed_len, *LIGHT_CLIENT_BOOTSTRAP_DENEB_MAX), ForkName::Electra => RpcLimits::new(altair_fixed_len, *LIGHT_CLIENT_BOOTSTRAP_ELECTRA_MAX), diff --git a/beacon_node/lighthouse_network/src/types/pubsub.rs b/beacon_node/lighthouse_network/src/types/pubsub.rs index a5cf03412d5..ed63ad014c9 100644 --- a/beacon_node/lighthouse_network/src/types/pubsub.rs +++ b/beacon_node/lighthouse_network/src/types/pubsub.rs @@ -10,8 +10,8 @@ use types::{ Attestation, AttesterSlashing, BlobSidecar, EthSpec, ForkContext, ForkName, LightClientFinalityUpdate, LightClientOptimisticUpdate, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBeaconBlockAltair, SignedBeaconBlockBase, - SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockElectra, - SignedBeaconBlockMerge, SignedBlsToExecutionChange, SignedContributionAndProof, + SignedBeaconBlockBellatrix, SignedBeaconBlockCapella, SignedBeaconBlockDeneb, + SignedBeaconBlockElectra, SignedBlsToExecutionChange, SignedContributionAndProof, SignedVoluntaryExit, SubnetId, SyncCommitteeMessage, SyncSubnetId, }; @@ -179,8 +179,8 @@ impl PubsubMessage { SignedBeaconBlockAltair::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?, ), - Some(ForkName::Merge) => SignedBeaconBlock::::Merge( - SignedBeaconBlockMerge::from_ssz_bytes(data) + Some(ForkName::Bellatrix) => SignedBeaconBlock::::Bellatrix( + SignedBeaconBlockBellatrix::from_ssz_bytes(data) .map_err(|e| format!("{:?}", e))?, ), Some(ForkName::Capella) => SignedBeaconBlock::::Capella( @@ -219,7 +219,7 @@ impl PubsubMessage { Some( ForkName::Base | ForkName::Altair - | ForkName::Merge + | ForkName::Bellatrix | ForkName::Capella, ) | None => Err(format!( diff --git a/beacon_node/lighthouse_network/src/types/topics.rs b/beacon_node/lighthouse_network/src/types/topics.rs index 7cbb5bac57f..c5f4b0c9ebb 100644 --- a/beacon_node/lighthouse_network/src/types/topics.rs +++ b/beacon_node/lighthouse_network/src/types/topics.rs @@ -47,7 +47,7 @@ pub fn fork_core_topics(fork_name: &ForkName, spec: &ChainSpec) -> V match fork_name { ForkName::Base => BASE_CORE_TOPICS.to_vec(), ForkName::Altair => ALTAIR_CORE_TOPICS.to_vec(), - ForkName::Merge => vec![], + ForkName::Bellatrix => vec![], ForkName::Capella => CAPELLA_CORE_TOPICS.to_vec(), ForkName::Deneb => { // All of deneb blob topics are core topics diff --git a/beacon_node/lighthouse_network/tests/common.rs b/beacon_node/lighthouse_network/tests/common.rs index 40ae341bd26..32e3a034666 100644 --- a/beacon_node/lighthouse_network/tests/common.rs +++ b/beacon_node/lighthouse_network/tests/common.rs @@ -21,13 +21,13 @@ use tempfile::Builder as TempBuilder; pub fn fork_context(fork_name: ForkName) -> ForkContext { let mut chain_spec = E::default_spec(); let altair_fork_epoch = Epoch::new(1); - let merge_fork_epoch = Epoch::new(2); + let bellatrix_fork_epoch = Epoch::new(2); let capella_fork_epoch = Epoch::new(3); let deneb_fork_epoch = Epoch::new(4); let electra_fork_epoch = Epoch::new(5); chain_spec.altair_fork_epoch = Some(altair_fork_epoch); - chain_spec.bellatrix_fork_epoch = Some(merge_fork_epoch); + chain_spec.bellatrix_fork_epoch = Some(bellatrix_fork_epoch); chain_spec.capella_fork_epoch = Some(capella_fork_epoch); chain_spec.deneb_fork_epoch = Some(deneb_fork_epoch); chain_spec.electra_fork_epoch = Some(electra_fork_epoch); @@ -35,7 +35,7 @@ pub fn fork_context(fork_name: ForkName) -> ForkContext { let current_slot = match fork_name { ForkName::Base => Slot::new(0), ForkName::Altair => altair_fork_epoch.start_slot(E::slots_per_epoch()), - ForkName::Merge => merge_fork_epoch.start_slot(E::slots_per_epoch()), + ForkName::Bellatrix => bellatrix_fork_epoch.start_slot(E::slots_per_epoch()), ForkName::Capella => capella_fork_epoch.start_slot(E::slots_per_epoch()), ForkName::Deneb => deneb_fork_epoch.start_slot(E::slots_per_epoch()), ForkName::Electra => electra_fork_epoch.start_slot(E::slots_per_epoch()), diff --git a/beacon_node/lighthouse_network/tests/rpc_tests.rs b/beacon_node/lighthouse_network/tests/rpc_tests.rs index 5a21b462d43..a60af4db3db 100644 --- a/beacon_node/lighthouse_network/tests/rpc_tests.rs +++ b/beacon_node/lighthouse_network/tests/rpc_tests.rs @@ -13,37 +13,37 @@ use std::time::Duration; use tokio::runtime::Runtime; use tokio::time::sleep; use types::{ - BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockMerge, BlobSidecar, ChainSpec, + BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockBellatrix, BlobSidecar, ChainSpec, EmptyBlock, Epoch, EthSpec, ForkContext, ForkName, Hash256, MinimalEthSpec, Signature, SignedBeaconBlock, Slot, }; type E = MinimalEthSpec; -/// Merge block with length < max_rpc_size. -fn merge_block_small(fork_context: &ForkContext, spec: &ChainSpec) -> BeaconBlock { - let mut block = BeaconBlockMerge::::empty(spec); +/// Bellatrix block with length < max_rpc_size. +fn bellatrix_block_small(fork_context: &ForkContext, spec: &ChainSpec) -> BeaconBlock { + let mut block = BeaconBlockBellatrix::::empty(spec); let tx = VariableList::from(vec![0; 1024]); let txs = VariableList::from(std::iter::repeat(tx).take(5000).collect::>()); block.body.execution_payload.execution_payload.transactions = txs; - let block = BeaconBlock::Merge(block); + let block = BeaconBlock::Bellatrix(block); assert!(block.ssz_bytes_len() <= max_rpc_size(fork_context, spec.max_chunk_size as usize)); block } -/// Merge block with length > MAX_RPC_SIZE. -/// The max limit for a merge block is in the order of ~16GiB which wouldn't fit in memory. -/// Hence, we generate a merge block just greater than `MAX_RPC_SIZE` to test rejection on the rpc layer. -fn merge_block_large(fork_context: &ForkContext, spec: &ChainSpec) -> BeaconBlock { - let mut block = BeaconBlockMerge::::empty(spec); +/// Bellatrix block with length > MAX_RPC_SIZE. +/// The max limit for a bellatrix block is in the order of ~16GiB which wouldn't fit in memory. +/// Hence, we generate a bellatrix block just greater than `MAX_RPC_SIZE` to test rejection on the rpc layer. +fn bellatrix_block_large(fork_context: &ForkContext, spec: &ChainSpec) -> BeaconBlock { + let mut block = BeaconBlockBellatrix::::empty(spec); let tx = VariableList::from(vec![0; 1024]); let txs = VariableList::from(std::iter::repeat(tx).take(100000).collect::>()); block.body.execution_payload.execution_payload.transactions = txs; - let block = BeaconBlock::Merge(block); + let block = BeaconBlock::Bellatrix(block); assert!(block.ssz_bytes_len() > max_rpc_size(fork_context, spec.max_chunk_size as usize)); block } @@ -167,7 +167,7 @@ fn test_tcp_blocks_by_range_chunked_rpc() { let (mut sender, mut receiver) = common::build_node_pair( Arc::downgrade(&rt), &log, - ForkName::Merge, + ForkName::Bellatrix, &spec, Protocol::Tcp, ) @@ -187,9 +187,10 @@ fn test_tcp_blocks_by_range_chunked_rpc() { let signed_full_block = SignedBeaconBlock::from_block(full_block, Signature::empty()); let rpc_response_altair = Response::BlocksByRange(Some(Arc::new(signed_full_block))); - let full_block = merge_block_small(&common::fork_context(ForkName::Merge), &spec); + let full_block = bellatrix_block_small(&common::fork_context(ForkName::Bellatrix), &spec); let signed_full_block = SignedBeaconBlock::from_block(full_block, Signature::empty()); - let rpc_response_merge_small = Response::BlocksByRange(Some(Arc::new(signed_full_block))); + let rpc_response_bellatrix_small = + Response::BlocksByRange(Some(Arc::new(signed_full_block))); // keep count of the number of messages received let mut messages_received = 0; @@ -216,7 +217,7 @@ fn test_tcp_blocks_by_range_chunked_rpc() { } else if messages_received < 4 { assert_eq!(response, rpc_response_altair.clone()); } else { - assert_eq!(response, rpc_response_merge_small.clone()); + assert_eq!(response, rpc_response_bellatrix_small.clone()); } messages_received += 1; warn!(log, "Chunk received"); @@ -249,13 +250,13 @@ fn test_tcp_blocks_by_range_chunked_rpc() { warn!(log, "Receiver got request"); for i in 0..messages_to_send { // Send first third of responses as base blocks, - // second as altair and third as merge. + // second as altair and third as bellatrix. let rpc_response = if i < 2 { rpc_response_base.clone() } else if i < 4 { rpc_response_altair.clone() } else { - rpc_response_merge_small.clone() + rpc_response_bellatrix_small.clone() }; receiver.send_response(peer_id, id, rpc_response.clone()); } @@ -368,7 +369,7 @@ fn test_blobs_by_range_chunked_rpc() { warn!(log, "Receiver got request"); for _ in 0..messages_to_send { // Send first third of responses as base blocks, - // second as altair and third as merge. + // second as altair and third as bellatrix. receiver.send_response(peer_id, id, rpc_response.clone()); } // send the stream termination @@ -411,7 +412,7 @@ fn test_tcp_blocks_by_range_over_limit() { let (mut sender, mut receiver) = common::build_node_pair( Arc::downgrade(&rt), &log, - ForkName::Merge, + ForkName::Bellatrix, &spec, Protocol::Tcp, ) @@ -421,9 +422,10 @@ fn test_tcp_blocks_by_range_over_limit() { let rpc_request = Request::BlocksByRange(BlocksByRangeRequest::new(0, messages_to_send)); // BlocksByRange Response - let full_block = merge_block_large(&common::fork_context(ForkName::Merge), &spec); + let full_block = bellatrix_block_large(&common::fork_context(ForkName::Bellatrix), &spec); let signed_full_block = SignedBeaconBlock::from_block(full_block, Signature::empty()); - let rpc_response_merge_large = Response::BlocksByRange(Some(Arc::new(signed_full_block))); + let rpc_response_bellatrix_large = + Response::BlocksByRange(Some(Arc::new(signed_full_block))); let request_id = messages_to_send as usize; // build the sender future @@ -458,7 +460,7 @@ fn test_tcp_blocks_by_range_over_limit() { // send the response warn!(log, "Receiver got request"); for _ in 0..messages_to_send { - let rpc_response = rpc_response_merge_large.clone(); + let rpc_response = rpc_response_bellatrix_large.clone(); receiver.send_response(peer_id, id, rpc_response.clone()); } // send the stream termination @@ -736,7 +738,7 @@ fn test_tcp_blocks_by_root_chunked_rpc() { let (mut sender, mut receiver) = common::build_node_pair( Arc::downgrade(&rt), &log, - ForkName::Merge, + ForkName::Bellatrix, &spec, Protocol::Tcp, ) @@ -764,9 +766,10 @@ fn test_tcp_blocks_by_root_chunked_rpc() { let signed_full_block = SignedBeaconBlock::from_block(full_block, Signature::empty()); let rpc_response_altair = Response::BlocksByRoot(Some(Arc::new(signed_full_block))); - let full_block = merge_block_small(&common::fork_context(ForkName::Merge), &spec); + let full_block = bellatrix_block_small(&common::fork_context(ForkName::Bellatrix), &spec); let signed_full_block = SignedBeaconBlock::from_block(full_block, Signature::empty()); - let rpc_response_merge_small = Response::BlocksByRoot(Some(Arc::new(signed_full_block))); + let rpc_response_bellatrix_small = + Response::BlocksByRoot(Some(Arc::new(signed_full_block))); // keep count of the number of messages received let mut messages_received = 0; @@ -790,7 +793,7 @@ fn test_tcp_blocks_by_root_chunked_rpc() { } else if messages_received < 4 { assert_eq!(response, rpc_response_altair.clone()); } else { - assert_eq!(response, rpc_response_merge_small.clone()); + assert_eq!(response, rpc_response_bellatrix_small.clone()); } messages_received += 1; debug!(log, "Chunk received"); @@ -822,13 +825,13 @@ fn test_tcp_blocks_by_root_chunked_rpc() { debug!(log, "Receiver got request"); for i in 0..messages_to_send { - // Send equal base, altair and merge blocks + // Send equal base, altair and bellatrix blocks let rpc_response = if i < 2 { rpc_response_base.clone() } else if i < 4 { rpc_response_altair.clone() } else { - rpc_response_merge_small.clone() + rpc_response_bellatrix_small.clone() }; receiver.send_response(peer_id, id, rpc_response); debug!(log, "Sending message"); diff --git a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs index 1e72dc42578..2a0c7ea089b 100644 --- a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs @@ -433,9 +433,10 @@ impl NetworkBeaconProcessor { ForkName::Deneb | ForkName::Electra => { self.chain.spec.max_request_blocks_deneb } - ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { - self.chain.spec.max_request_blocks - } + ForkName::Base + | ForkName::Altair + | ForkName::Bellatrix + | ForkName::Capella => self.chain.spec.max_request_blocks, } }); if *req.count() > max_request_size { diff --git a/beacon_node/store/src/impls/execution_payload.rs b/beacon_node/store/src/impls/execution_payload.rs index a874031ca27..14fc10ad6de 100644 --- a/beacon_node/store/src/impls/execution_payload.rs +++ b/beacon_node/store/src/impls/execution_payload.rs @@ -1,8 +1,8 @@ use crate::{DBColumn, Error, StoreItem}; use ssz::{Decode, Encode}; use types::{ - BlobSidecarList, EthSpec, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, - ExecutionPayloadElectra, ExecutionPayloadMerge, + BlobSidecarList, EthSpec, ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella, + ExecutionPayloadDeneb, ExecutionPayloadElectra, }; macro_rules! impl_store_item { @@ -22,7 +22,7 @@ macro_rules! impl_store_item { } }; } -impl_store_item!(ExecutionPayloadMerge); +impl_store_item!(ExecutionPayloadBellatrix); impl_store_item!(ExecutionPayloadCapella); impl_store_item!(ExecutionPayloadDeneb); impl_store_item!(ExecutionPayloadElectra); @@ -51,7 +51,8 @@ impl StoreItem for ExecutionPayload { ExecutionPayloadCapella::from_ssz_bytes(bytes) .map(Self::Capella) .or_else(|_| { - ExecutionPayloadMerge::from_ssz_bytes(bytes).map(Self::Merge) + ExecutionPayloadBellatrix::from_ssz_bytes(bytes) + .map(Self::Bellatrix) }) }) }) diff --git a/beacon_node/store/src/partial_beacon_state.rs b/beacon_node/store/src/partial_beacon_state.rs index 25438fc7e0a..c2a15c0266f 100644 --- a/beacon_node/store/src/partial_beacon_state.rs +++ b/beacon_node/store/src/partial_beacon_state.rs @@ -14,7 +14,7 @@ use types::*; /// /// Utilises lazy-loading from separate storage for its vector fields. #[superstruct( - variants(Base, Altair, Merge, Capella, Deneb, Electra), + variants(Base, Altair, Bellatrix, Capella, Deneb, Electra), variant_attributes(derive(Debug, PartialEq, Clone, Encode, Decode)) )] #[derive(Debug, PartialEq, Clone, Encode)] @@ -66,9 +66,9 @@ where pub current_epoch_attestations: List, E::MaxPendingAttestations>, // Participation (Altair and later) - #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] pub previous_epoch_participation: List, - #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] pub current_epoch_participation: List, // Finality @@ -78,21 +78,21 @@ where pub finalized_checkpoint: Checkpoint, // Inactivity - #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] pub inactivity_scores: List, // Light-client sync committees - #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] pub current_sync_committee: Arc>, - #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] pub next_sync_committee: Arc>, // Execution #[superstruct( - only(Merge), - partial_getter(rename = "latest_execution_payload_header_merge") + only(Bellatrix), + partial_getter(rename = "latest_execution_payload_header_bellatrix") )] - pub latest_execution_payload_header: ExecutionPayloadHeaderMerge, + pub latest_execution_payload_header: ExecutionPayloadHeaderBellatrix, #[superstruct( only(Capella), partial_getter(rename = "latest_execution_payload_header_capella") @@ -199,11 +199,11 @@ impl PartialBeaconState { ], [] ), - BeaconState::Merge(s) => impl_from_state_forgetful!( + BeaconState::Bellatrix(s) => impl_from_state_forgetful!( s, outer, - Merge, - PartialBeaconStateMerge, + Bellatrix, + PartialBeaconStateBellatrix, [ previous_epoch_participation, current_epoch_participation, @@ -467,10 +467,10 @@ impl TryInto> for PartialBeaconState { ], [] ), - PartialBeaconState::Merge(inner) => impl_try_into_beacon_state!( + PartialBeaconState::Bellatrix(inner) => impl_try_into_beacon_state!( inner, - Merge, - BeaconStateMerge, + Bellatrix, + BeaconStateBellatrix, [ previous_epoch_participation, current_epoch_participation, diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index feff1d391a9..838be4beffb 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1023,7 +1023,7 @@ impl ForkVersionDeserialize for SsePayloadAttributes { fork_name: ForkName, ) -> Result { match fork_name { - ForkName::Merge => serde_json::from_value(value) + ForkName::Bellatrix => serde_json::from_value(value) .map(Self::V1) .map_err(serde::de::Error::custom), ForkName::Capella => serde_json::from_value(value) @@ -1598,7 +1598,7 @@ impl FullBlockContents { fork_name: ForkName, ) -> Result { match fork_name { - ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => { BeaconBlock::from_ssz_bytes_for_fork(bytes, fork_name) .map(|block| FullBlockContents::Block(block)) } @@ -1658,7 +1658,7 @@ impl ForkVersionDeserialize for FullBlockContents { fork_name: ForkName, ) -> Result { match fork_name { - ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => { Ok(FullBlockContents::Block( BeaconBlock::deserialize_by_fork::<'de, D>(value, fork_name)?, )) @@ -1758,7 +1758,7 @@ impl PublishBlockRequest { /// SSZ decode with fork variant determined by `fork_name`. pub fn from_ssz_bytes(bytes: &[u8], fork_name: ForkName) -> Result { match fork_name { - ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => { SignedBeaconBlock::from_ssz_bytes_for_fork(bytes, fork_name) .map(|block| PublishBlockRequest::Block(Arc::new(block))) } @@ -1844,7 +1844,7 @@ impl TryFrom>> for PublishBlockRequest { match *block { SignedBeaconBlock::Base(_) | SignedBeaconBlock::Altair(_) - | SignedBeaconBlock::Merge(_) + | SignedBeaconBlock::Bellatrix(_) | SignedBeaconBlock::Capella(_) => Ok(PublishBlockRequest::Block(block)), SignedBeaconBlock::Deneb(_) | SignedBeaconBlock::Electra(_) => Err( "post-Deneb block contents cannot be fully constructed from just the signed block", @@ -1953,7 +1953,7 @@ impl ForkVersionDeserialize for FullPayloadContents { fork_name: ForkName, ) -> Result { match fork_name { - ForkName::Merge | ForkName::Capella => serde_json::from_value(value) + ForkName::Bellatrix | ForkName::Capella => serde_json::from_value(value) .map(Self::Payload) .map_err(serde::de::Error::custom), ForkName::Deneb | ForkName::Electra => serde_json::from_value(value) diff --git a/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml b/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml index 27fb81a5139..50a5fcc3a50 100644 --- a/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml @@ -33,7 +33,7 @@ GENESIS_DELAY: 6000 # Altair ALTAIR_FORK_VERSION: 0x01000064 ALTAIR_FORK_EPOCH: 512 -# Merge +# Bellatrix BELLATRIX_FORK_VERSION: 0x02000064 BELLATRIX_FORK_EPOCH: 385536 # Capella diff --git a/common/eth2_network_config/built_in_network_configs/holesky/config.yaml b/common/eth2_network_config/built_in_network_configs/holesky/config.yaml index bd384cfe497..6a399b957d2 100644 --- a/common/eth2_network_config/built_in_network_configs/holesky/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/holesky/config.yaml @@ -22,7 +22,7 @@ GENESIS_DELAY: 300 # Altair ALTAIR_FORK_VERSION: 0x02017000 ALTAIR_FORK_EPOCH: 0 -# Merge +# Bellatrix BELLATRIX_FORK_VERSION: 0x03017000 BELLATRIX_FORK_EPOCH: 0 TERMINAL_TOTAL_DIFFICULTY: 0 diff --git a/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml b/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml index 2df40798c11..72a48679118 100644 --- a/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/sepolia/config.yaml @@ -21,7 +21,7 @@ GENESIS_DELAY: 86400 ALTAIR_FORK_VERSION: 0x90000070 ALTAIR_FORK_EPOCH: 50 -# Merge +# Bellatrix BELLATRIX_FORK_VERSION: 0x90000071 BELLATRIX_FORK_EPOCH: 100 TERMINAL_TOTAL_DIFFICULTY: 17000000000000000 diff --git a/common/slot_clock/src/lib.rs b/common/slot_clock/src/lib.rs index 6bf74645000..4f54b2ee76b 100644 --- a/common/slot_clock/src/lib.rs +++ b/common/slot_clock/src/lib.rs @@ -11,7 +11,7 @@ pub use crate::manual_slot_clock::ManualSlotClock as TestingSlotClock; pub use crate::manual_slot_clock::ManualSlotClock; pub use crate::system_time_slot_clock::SystemTimeSlotClock; pub use metrics::scrape_for_metrics; -use types::consts::merge::INTERVALS_PER_SLOT; +use types::consts::bellatrix::INTERVALS_PER_SLOT; pub use types::Slot; /// A clock that reports the current slot. diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index 6e3f6717ede..2846a0112cd 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -13,7 +13,7 @@ use std::collections::BTreeSet; use std::marker::PhantomData; use std::time::Duration; use types::{ - consts::merge::INTERVALS_PER_SLOT, AbstractExecPayload, AttestationShufflingId, + consts::bellatrix::INTERVALS_PER_SLOT, AbstractExecPayload, AttestationShufflingId, AttesterSlashing, BeaconBlockRef, BeaconState, BeaconStateError, ChainSpec, Checkpoint, Epoch, EthSpec, ExecPayload, ExecutionBlockHash, Hash256, IndexedAttestation, RelativeEpoch, SignedBeaconBlock, Slot, @@ -751,7 +751,7 @@ where BeaconBlockRef::Electra(_) | BeaconBlockRef::Deneb(_) | BeaconBlockRef::Capella(_) - | BeaconBlockRef::Merge(_) + | BeaconBlockRef::Bellatrix(_) | BeaconBlockRef::Altair(_) => { // NOTE: Processing justification & finalization requires the progressive // balances cache, but we cannot initialize it here as we only have an diff --git a/consensus/state_processing/src/common/get_attestation_participation.rs b/consensus/state_processing/src/common/get_attestation_participation.rs index d27a00c3826..fc09dad1f4e 100644 --- a/consensus/state_processing/src/common/get_attestation_participation.rs +++ b/consensus/state_processing/src/common/get_attestation_participation.rs @@ -47,7 +47,7 @@ pub fn get_attestation_participation_flag_indices( match state { &BeaconState::Base(_) | &BeaconState::Altair(_) - | &BeaconState::Merge(_) + | &BeaconState::Bellatrix(_) | &BeaconState::Capella(_) => { if is_matching_target && inclusion_delay <= E::slots_per_epoch() { participation_flag_indices.push(TIMELY_TARGET_FLAG_INDEX); diff --git a/consensus/state_processing/src/common/slash_validator.rs b/consensus/state_processing/src/common/slash_validator.rs index 16b4e74ece9..520b58a8af3 100644 --- a/consensus/state_processing/src/common/slash_validator.rs +++ b/consensus/state_processing/src/common/slash_validator.rs @@ -58,7 +58,7 @@ pub fn slash_validator( let proposer_reward = match state { BeaconState::Base(_) => whistleblower_reward.safe_div(spec.proposer_reward_quotient)?, BeaconState::Altair(_) - | BeaconState::Merge(_) + | BeaconState::Bellatrix(_) | BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) => whistleblower_reward diff --git a/consensus/state_processing/src/genesis.rs b/consensus/state_processing/src/genesis.rs index 5ab9a99a3b1..a84f359389c 100644 --- a/consensus/state_processing/src/genesis.rs +++ b/consensus/state_processing/src/genesis.rs @@ -63,7 +63,7 @@ pub fn initialize_beacon_state_from_eth1( .bellatrix_fork_epoch .map_or(false, |fork_epoch| fork_epoch == E::genesis_epoch()) { - // this will set state.latest_execution_payload_header = ExecutionPayloadHeaderMerge::default() + // this will set state.latest_execution_payload_header = ExecutionPayloadHeaderBellatrix::default() upgrade_to_bellatrix(&mut state, spec)?; // Remove intermediate Altair fork from `state.fork`. @@ -71,8 +71,8 @@ pub fn initialize_beacon_state_from_eth1( // Override latest execution payload header. // See https://github.com/ethereum/consensus-specs/blob/v1.1.0/specs/bellatrix/beacon-chain.md#testing - if let Some(ExecutionPayloadHeader::Merge(ref header)) = execution_payload_header { - *state.latest_execution_payload_header_merge_mut()? = header.clone(); + if let Some(ExecutionPayloadHeader::Bellatrix(ref header)) = execution_payload_header { + *state.latest_execution_payload_header_bellatrix_mut()? = header.clone(); } } diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index a8447b7714a..2efa1218829 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -418,9 +418,9 @@ pub fn process_execution_payload>( partially_verify_execution_payload::(state, state.slot(), body, spec)?; let payload = body.execution_payload()?; match state.latest_execution_payload_header_mut()? { - ExecutionPayloadHeaderRefMut::Merge(header_mut) => { + ExecutionPayloadHeaderRefMut::Bellatrix(header_mut) => { match payload.to_execution_payload_header() { - ExecutionPayloadHeader::Merge(header) => *header_mut = header, + ExecutionPayloadHeader::Bellatrix(header) => *header_mut = header, _ => return Err(BlockProcessingError::IncorrectStateType), } } @@ -449,14 +449,14 @@ pub fn process_execution_payload>( /// These functions will definitely be called before the merge. Their entire purpose is to check if /// the merge has happened or if we're on the transition block. Thus we don't want to propagate -/// errors from the `BeaconState` being an earlier variant than `BeaconStateMerge` as we'd have to +/// errors from the `BeaconState` being an earlier variant than `BeaconStateBellatrix` as we'd have to /// repeatedly write code to treat these errors as false. /// https://github.com/ethereum/consensus-specs/blob/dev/specs/bellatrix/beacon-chain.md#is_merge_transition_complete pub fn is_merge_transition_complete(state: &BeaconState) -> bool { match state { // We must check defaultness against the payload header with 0x0 roots, as that's what's meant // by `ExecutionPayloadHeader()` in the spec. - BeaconState::Merge(_) => state + BeaconState::Bellatrix(_) => state .latest_execution_payload_header() .map(|header| !header.is_default_with_zero_roots()) .unwrap_or(false), @@ -557,7 +557,7 @@ pub fn process_withdrawals>( spec: &ChainSpec, ) -> Result<(), BlockProcessingError> { match state { - BeaconState::Merge(_) => Ok(()), + BeaconState::Bellatrix(_) => Ok(()), BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) => { let expected_withdrawals = get_expected_withdrawals(state, spec)?; let expected_root = expected_withdrawals.tree_hash_root(); diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index d812306a0f7..3aefcf8a9c5 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -273,7 +273,7 @@ pub fn process_attestations>( )?; } BeaconBlockBodyRef::Altair(_) - | BeaconBlockBodyRef::Merge(_) + | BeaconBlockBodyRef::Bellatrix(_) | BeaconBlockBodyRef::Capella(_) | BeaconBlockBodyRef::Deneb(_) | BeaconBlockBodyRef::Electra(_) => { diff --git a/consensus/state_processing/src/per_block_processing/signature_sets.rs b/consensus/state_processing/src/per_block_processing/signature_sets.rs index 163b2cff7a9..9468893f762 100644 --- a/consensus/state_processing/src/per_block_processing/signature_sets.rs +++ b/consensus/state_processing/src/per_block_processing/signature_sets.rs @@ -390,7 +390,7 @@ where let domain = match state { BeaconState::Base(_) | BeaconState::Altair(_) - | BeaconState::Merge(_) + | BeaconState::Bellatrix(_) | BeaconState::Capella(_) => spec.get_domain( exit.epoch, Domain::VoluntaryExit, diff --git a/consensus/state_processing/src/per_block_processing/verify_attestation.rs b/consensus/state_processing/src/per_block_processing/verify_attestation.rs index 73454559dfd..c904ba55f0a 100644 --- a/consensus/state_processing/src/per_block_processing/verify_attestation.rs +++ b/consensus/state_processing/src/per_block_processing/verify_attestation.rs @@ -35,7 +35,7 @@ pub fn verify_attestation_for_block_inclusion<'ctxt, E: EthSpec>( match state { BeaconState::Base(_) | BeaconState::Altair(_) - | BeaconState::Merge(_) + | BeaconState::Bellatrix(_) | BeaconState::Capella(_) => { verify!( state.slot() <= data.slot.safe_add(E::slots_per_epoch())?, diff --git a/consensus/state_processing/src/per_epoch_processing.rs b/consensus/state_processing/src/per_epoch_processing.rs index b51aa23f370..55e8853f3f8 100644 --- a/consensus/state_processing/src/per_epoch_processing.rs +++ b/consensus/state_processing/src/per_epoch_processing.rs @@ -44,7 +44,7 @@ pub fn process_epoch( match state { BeaconState::Base(_) => base::process_epoch(state, spec), BeaconState::Altair(_) - | BeaconState::Merge(_) + | BeaconState::Bellatrix(_) | BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) => altair::process_epoch(state, spec), diff --git a/consensus/state_processing/src/per_slot_processing.rs b/consensus/state_processing/src/per_slot_processing.rs index cc28340962a..6554423199f 100644 --- a/consensus/state_processing/src/per_slot_processing.rs +++ b/consensus/state_processing/src/per_slot_processing.rs @@ -54,7 +54,7 @@ pub fn per_slot_processing( if spec.altair_fork_epoch == Some(state.current_epoch()) { upgrade_to_altair(state, spec)?; } - // If the Merge fork epoch is reached, perform an irregular state upgrade. + // If the Bellatrix fork epoch is reached, perform an irregular state upgrade. if spec.bellatrix_fork_epoch == Some(state.current_epoch()) { upgrade_to_bellatrix(state, spec)?; } diff --git a/consensus/state_processing/src/upgrade.rs b/consensus/state_processing/src/upgrade.rs index 98602c66ba3..93cafa73d03 100644 --- a/consensus/state_processing/src/upgrade.rs +++ b/consensus/state_processing/src/upgrade.rs @@ -1,11 +1,11 @@ pub mod altair; +pub mod bellatrix; pub mod capella; pub mod deneb; pub mod electra; -pub mod merge; pub use altair::upgrade_to_altair; +pub use bellatrix::upgrade_to_bellatrix; pub use capella::upgrade_to_capella; pub use deneb::upgrade_to_deneb; pub use electra::upgrade_to_electra; -pub use merge::upgrade_to_bellatrix; diff --git a/consensus/state_processing/src/upgrade/merge.rs b/consensus/state_processing/src/upgrade/bellatrix.rs similarity index 90% rename from consensus/state_processing/src/upgrade/merge.rs rename to consensus/state_processing/src/upgrade/bellatrix.rs index 02705743ceb..f23e571cd12 100644 --- a/consensus/state_processing/src/upgrade/merge.rs +++ b/consensus/state_processing/src/upgrade/bellatrix.rs @@ -1,10 +1,10 @@ use std::mem; use types::{ - BeaconState, BeaconStateError as Error, BeaconStateMerge, ChainSpec, EpochCache, EthSpec, - ExecutionPayloadHeaderMerge, Fork, + BeaconState, BeaconStateBellatrix, BeaconStateError as Error, ChainSpec, EpochCache, EthSpec, + ExecutionPayloadHeaderBellatrix, Fork, }; -/// Transform a `Altair` state into an `Merge` state. +/// Transform a `Altair` state into an `Bellatrix` state. pub fn upgrade_to_bellatrix( pre_state: &mut BeaconState, spec: &ChainSpec, @@ -17,7 +17,7 @@ pub fn upgrade_to_bellatrix( // // Fixed size vectors get cloned because replacing them would require the same size // allocation as cloning. - let post = BeaconState::Merge(BeaconStateMerge { + let post = BeaconState::Bellatrix(BeaconStateBellatrix { // Versioning genesis_time: pre.genesis_time, genesis_validators_root: pre.genesis_validators_root, @@ -57,7 +57,7 @@ pub fn upgrade_to_bellatrix( current_sync_committee: pre.current_sync_committee.clone(), next_sync_committee: pre.next_sync_committee.clone(), // Execution - latest_execution_payload_header: >::default(), + latest_execution_payload_header: >::default(), // Caches total_active_balance: pre.total_active_balance, progressive_balances_cache: mem::take(&mut pre.progressive_balances_cache), diff --git a/consensus/state_processing/src/upgrade/capella.rs b/consensus/state_processing/src/upgrade/capella.rs index 51e29d10f3c..ae0dbde7678 100644 --- a/consensus/state_processing/src/upgrade/capella.rs +++ b/consensus/state_processing/src/upgrade/capella.rs @@ -4,13 +4,13 @@ use types::{ Fork, List, }; -/// Transform a `Merge` state into an `Capella` state. +/// Transform a `Bellatrix` state into an `Capella` state. pub fn upgrade_to_capella( pre_state: &mut BeaconState, spec: &ChainSpec, ) -> Result<(), Error> { let epoch = pre_state.current_epoch(); - let pre = pre_state.as_merge_mut()?; + let pre = pre_state.as_bellatrix_mut()?; // Where possible, use something like `mem::take` to move fields from behind the &mut // reference. For other fields that don't have a good default value, use `clone`. diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index 14874f0204f..5af53b3fa17 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -12,7 +12,7 @@ use tree_hash_derive::TreeHash; /// A block of the `BeaconChain`. #[superstruct( - variants(Base, Altair, Merge, Capella, Deneb, Electra), + variants(Base, Altair, Bellatrix, Capella, Deneb, Electra), variant_attributes( derive( Debug, @@ -63,8 +63,8 @@ pub struct BeaconBlock = FullPayload pub body: BeaconBlockBodyBase, #[superstruct(only(Altair), partial_getter(rename = "body_altair"))] pub body: BeaconBlockBodyAltair, - #[superstruct(only(Merge), partial_getter(rename = "body_merge"))] - pub body: BeaconBlockBodyMerge, + #[superstruct(only(Bellatrix), partial_getter(rename = "body_bellatrix"))] + pub body: BeaconBlockBodyBellatrix, #[superstruct(only(Capella), partial_getter(rename = "body_capella"))] pub body: BeaconBlockBodyCapella, #[superstruct(only(Deneb), partial_getter(rename = "body_deneb"))] @@ -130,7 +130,7 @@ impl> BeaconBlock { .map(BeaconBlock::Electra) .or_else(|_| BeaconBlockDeneb::from_ssz_bytes(bytes).map(BeaconBlock::Deneb)) .or_else(|_| BeaconBlockCapella::from_ssz_bytes(bytes).map(BeaconBlock::Capella)) - .or_else(|_| BeaconBlockMerge::from_ssz_bytes(bytes).map(BeaconBlock::Merge)) + .or_else(|_| BeaconBlockBellatrix::from_ssz_bytes(bytes).map(BeaconBlock::Bellatrix)) .or_else(|_| BeaconBlockAltair::from_ssz_bytes(bytes).map(BeaconBlock::Altair)) .or_else(|_| BeaconBlockBase::from_ssz_bytes(bytes).map(BeaconBlock::Base)) } @@ -221,7 +221,7 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockRef<'a, E, Payl match self { BeaconBlockRef::Base { .. } => ForkName::Base, BeaconBlockRef::Altair { .. } => ForkName::Altair, - BeaconBlockRef::Merge { .. } => ForkName::Merge, + BeaconBlockRef::Bellatrix { .. } => ForkName::Bellatrix, BeaconBlockRef::Capella { .. } => ForkName::Capella, BeaconBlockRef::Deneb { .. } => ForkName::Deneb, BeaconBlockRef::Electra { .. } => ForkName::Electra, @@ -466,15 +466,15 @@ impl> BeaconBlockAltair } } -impl> EmptyBlock for BeaconBlockMerge { - /// Returns an empty Merge block to be used during genesis. +impl> EmptyBlock for BeaconBlockBellatrix { + /// Returns an empty Bellatrix block to be used during genesis. fn empty(spec: &ChainSpec) -> Self { - BeaconBlockMerge { + BeaconBlockBellatrix { slot: spec.genesis_slot, proposer_index: 0, parent_root: Hash256::zero(), state_root: Hash256::zero(), - body: BeaconBlockBodyMerge { + body: BeaconBlockBodyBellatrix { randao_reveal: Signature::empty(), eth1_data: Eth1Data { deposit_root: Hash256::zero(), @@ -488,7 +488,7 @@ impl> EmptyBlock for BeaconBlockMerg deposits: VariableList::empty(), voluntary_exits: VariableList::empty(), sync_aggregate: SyncAggregate::empty(), - execution_payload: Payload::Merge::default(), + execution_payload: Payload::Bellatrix::default(), }, } } @@ -753,7 +753,7 @@ macro_rules! impl_from { impl_from!(BeaconBlockBase, >, >, |body: BeaconBlockBodyBase<_, _>| body.into()); impl_from!(BeaconBlockAltair, >, >, |body: BeaconBlockBodyAltair<_, _>| body.into()); -impl_from!(BeaconBlockMerge, >, >, |body: BeaconBlockBodyMerge<_, _>| body.into()); +impl_from!(BeaconBlockBellatrix, >, >, |body: BeaconBlockBodyBellatrix<_, _>| body.into()); impl_from!(BeaconBlockCapella, >, >, |body: BeaconBlockBodyCapella<_, _>| body.into()); impl_from!(BeaconBlockDeneb, >, >, |body: BeaconBlockBodyDeneb<_, _>| body.into()); impl_from!(BeaconBlockElectra, >, >, |body: BeaconBlockBodyElectra<_, _>| body.into()); @@ -786,7 +786,7 @@ macro_rules! impl_clone_as_blinded { impl_clone_as_blinded!(BeaconBlockBase, >, >); impl_clone_as_blinded!(BeaconBlockAltair, >, >); -impl_clone_as_blinded!(BeaconBlockMerge, >, >); +impl_clone_as_blinded!(BeaconBlockBellatrix, >, >); impl_clone_as_blinded!(BeaconBlockCapella, >, >); impl_clone_as_blinded!(BeaconBlockDeneb, >, >); impl_clone_as_blinded!(BeaconBlockElectra, >, >); diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index a55c16b80d5..c3077c4ab68 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -29,7 +29,7 @@ pub const BLOB_KZG_COMMITMENTS_INDEX: usize = 11; /// /// This *superstruct* abstracts over the hard-fork. #[superstruct( - variants(Base, Altair, Merge, Capella, Deneb, Electra), + variants(Base, Altair, Bellatrix, Capella, Deneb, Electra), variant_attributes( derive( Debug, @@ -67,14 +67,17 @@ pub struct BeaconBlockBody = FullPay pub attestations: VariableList, E::MaxAttestations>, pub deposits: VariableList, pub voluntary_exits: VariableList, - #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] pub sync_aggregate: SyncAggregate, // We flatten the execution payload so that serde can use the name of the inner type, // either `execution_payload` for full payloads, or `execution_payload_header` for blinded // payloads. - #[superstruct(only(Merge), partial_getter(rename = "execution_payload_merge"))] + #[superstruct( + only(Bellatrix), + partial_getter(rename = "execution_payload_bellatrix") + )] #[serde(flatten)] - pub execution_payload: Payload::Merge, + pub execution_payload: Payload::Bellatrix, #[superstruct(only(Capella), partial_getter(rename = "execution_payload_capella"))] #[serde(flatten)] pub execution_payload: Payload::Capella, @@ -107,7 +110,7 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, pub fn execution_payload(&self) -> Result, Error> { match self { Self::Base(_) | Self::Altair(_) => Err(Error::IncorrectStateVariant), - Self::Merge(body) => Ok(Payload::Ref::from(&body.execution_payload)), + Self::Bellatrix(body) => Ok(Payload::Ref::from(&body.execution_payload)), Self::Capella(body) => Ok(Payload::Ref::from(&body.execution_payload)), Self::Deneb(body) => Ok(Payload::Ref::from(&body.execution_payload)), Self::Electra(body) => Ok(Payload::Ref::from(&body.execution_payload)), @@ -121,7 +124,7 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, index: usize, ) -> Result, Error> { match self { - Self::Base(_) | Self::Altair(_) | Self::Merge(_) | Self::Capella(_) => { + Self::Base(_) | Self::Altair(_) | Self::Bellatrix(_) | Self::Capella(_) => { Err(Error::IncorrectStateVariant) } Self::Deneb(body) => { @@ -261,7 +264,7 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, E, match self { BeaconBlockBodyRef::Base { .. } => ForkName::Base, BeaconBlockBodyRef::Altair { .. } => ForkName::Altair, - BeaconBlockBodyRef::Merge { .. } => ForkName::Merge, + BeaconBlockBodyRef::Bellatrix { .. } => ForkName::Bellatrix, BeaconBlockBodyRef::Capella { .. } => ForkName::Capella, BeaconBlockBodyRef::Deneb { .. } => ForkName::Deneb, BeaconBlockBodyRef::Electra { .. } => ForkName::Electra, @@ -407,14 +410,14 @@ impl From>> } } -impl From>> +impl From>> for ( - BeaconBlockBodyMerge>, - Option>, + BeaconBlockBodyBellatrix>, + Option>, ) { - fn from(body: BeaconBlockBodyMerge>) -> Self { - let BeaconBlockBodyMerge { + fn from(body: BeaconBlockBodyBellatrix>) -> Self { + let BeaconBlockBodyBellatrix { randao_reveal, eth1_data, graffiti, @@ -424,11 +427,11 @@ impl From>> deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayloadMerge { execution_payload }, + execution_payload: FullPayloadBellatrix { execution_payload }, } = body; ( - BeaconBlockBodyMerge { + BeaconBlockBodyBellatrix { randao_reveal, eth1_data, graffiti, @@ -438,7 +441,7 @@ impl From>> deposits, voluntary_exits, sync_aggregate, - execution_payload: BlindedPayloadMerge { + execution_payload: BlindedPayloadBellatrix { execution_payload_header: From::from(&execution_payload), }, }, @@ -592,9 +595,9 @@ impl BeaconBlockBodyAltair> { } } -impl BeaconBlockBodyMerge> { - pub fn clone_as_blinded(&self) -> BeaconBlockBodyMerge> { - let BeaconBlockBodyMerge { +impl BeaconBlockBodyBellatrix> { + pub fn clone_as_blinded(&self) -> BeaconBlockBodyBellatrix> { + let BeaconBlockBodyBellatrix { randao_reveal, eth1_data, graffiti, @@ -604,10 +607,10 @@ impl BeaconBlockBodyMerge> { deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayloadMerge { execution_payload }, + execution_payload: FullPayloadBellatrix { execution_payload }, } = self; - BeaconBlockBodyMerge { + BeaconBlockBodyBellatrix { randao_reveal: randao_reveal.clone(), eth1_data: eth1_data.clone(), graffiti: *graffiti, @@ -617,7 +620,7 @@ impl BeaconBlockBodyMerge> { deposits: deposits.clone(), voluntary_exits: voluntary_exits.clone(), sync_aggregate: sync_aggregate.clone(), - execution_payload: BlindedPayloadMerge { + execution_payload: BlindedPayloadBellatrix { execution_payload_header: execution_payload.into(), }, } diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index 5da81f6a752..f6bae53505b 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -206,7 +206,7 @@ impl From for Hash256 { /// The state of the `BeaconChain` at some slot. #[superstruct( - variants(Base, Altair, Merge, Capella, Deneb, Electra), + variants(Base, Altair, Bellatrix, Capella, Deneb, Electra), variant_attributes( derive( Derivative, @@ -254,14 +254,14 @@ impl From for Hash256 { )), num_fields(all()), )), - Merge(metastruct( + Bellatrix(metastruct( mappings( map_beacon_state_bellatrix_fields(), map_beacon_state_bellatrix_tree_list_fields(mutable, fallible, groups(tree_lists)), map_beacon_state_bellatrix_tree_list_fields_immutable(groups(tree_lists)), ), bimappings(bimap_beacon_state_bellatrix_tree_list_fields( - other_type = "BeaconStateMerge", + other_type = "BeaconStateBellatrix", self_mutable, fallible, groups(tree_lists) @@ -392,10 +392,10 @@ where pub current_epoch_attestations: List, E::MaxPendingAttestations>, // Participation (Altair and later) - #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] #[test_random(default)] pub previous_epoch_participation: List, - #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] #[test_random(default)] pub current_epoch_participation: List, @@ -415,25 +415,25 @@ where // Inactivity #[serde(with = "ssz_types::serde_utils::quoted_u64_var_list")] - #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] #[test_random(default)] pub inactivity_scores: List, // Light-client sync committees - #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] #[metastruct(exclude_from(tree_lists))] pub current_sync_committee: Arc>, - #[superstruct(only(Altair, Merge, Capella, Deneb, Electra))] + #[superstruct(only(Altair, Bellatrix, Capella, Deneb, Electra))] #[metastruct(exclude_from(tree_lists))] pub next_sync_committee: Arc>, // Execution #[superstruct( - only(Merge), - partial_getter(rename = "latest_execution_payload_header_merge") + only(Bellatrix), + partial_getter(rename = "latest_execution_payload_header_bellatrix") )] #[metastruct(exclude_from(tree_lists))] - pub latest_execution_payload_header: ExecutionPayloadHeaderMerge, + pub latest_execution_payload_header: ExecutionPayloadHeaderBellatrix, #[superstruct( only(Capella), partial_getter(rename = "latest_execution_payload_header_capella") @@ -601,7 +601,7 @@ impl BeaconState { match self { BeaconState::Base { .. } => ForkName::Base, BeaconState::Altair { .. } => ForkName::Altair, - BeaconState::Merge { .. } => ForkName::Merge, + BeaconState::Bellatrix { .. } => ForkName::Bellatrix, BeaconState::Capella { .. } => ForkName::Capella, BeaconState::Deneb { .. } => ForkName::Deneb, BeaconState::Electra { .. } => ForkName::Electra, @@ -884,7 +884,7 @@ impl BeaconState { pub fn latest_execution_payload_header(&self) -> Result, Error> { match self { BeaconState::Base(_) | BeaconState::Altair(_) => Err(Error::IncorrectStateVariant), - BeaconState::Merge(state) => Ok(ExecutionPayloadHeaderRef::Merge( + BeaconState::Bellatrix(state) => Ok(ExecutionPayloadHeaderRef::Bellatrix( &state.latest_execution_payload_header, )), BeaconState::Capella(state) => Ok(ExecutionPayloadHeaderRef::Capella( @@ -904,7 +904,7 @@ impl BeaconState { ) -> Result, Error> { match self { BeaconState::Base(_) | BeaconState::Altair(_) => Err(Error::IncorrectStateVariant), - BeaconState::Merge(state) => Ok(ExecutionPayloadHeaderRefMut::Merge( + BeaconState::Bellatrix(state) => Ok(ExecutionPayloadHeaderRefMut::Bellatrix( &mut state.latest_execution_payload_header, )), BeaconState::Capella(state) => Ok(ExecutionPayloadHeaderRefMut::Capella( @@ -1390,7 +1390,7 @@ impl BeaconState { &mut state.exit_cache, &mut state.epoch_cache, )), - BeaconState::Merge(state) => Ok(( + BeaconState::Bellatrix(state) => Ok(( &mut state.validators, &mut state.balances, &state.previous_epoch_participation, @@ -1555,7 +1555,7 @@ impl BeaconState { Ok(match self { BeaconState::Base(_) | BeaconState::Altair(_) - | BeaconState::Merge(_) + | BeaconState::Bellatrix(_) | BeaconState::Capella(_) => self.get_validator_churn_limit(spec)?, BeaconState::Deneb(_) | BeaconState::Electra(_) => std::cmp::min( spec.max_per_epoch_activation_churn_limit, @@ -1674,7 +1674,7 @@ impl BeaconState { match self { BeaconState::Base(_) => Err(BeaconStateError::IncorrectStateVariant), BeaconState::Altair(state) => Ok(&mut state.current_epoch_participation), - BeaconState::Merge(state) => Ok(&mut state.current_epoch_participation), + BeaconState::Bellatrix(state) => Ok(&mut state.current_epoch_participation), BeaconState::Capella(state) => Ok(&mut state.current_epoch_participation), BeaconState::Deneb(state) => Ok(&mut state.current_epoch_participation), BeaconState::Electra(state) => Ok(&mut state.current_epoch_participation), @@ -1683,7 +1683,7 @@ impl BeaconState { match self { BeaconState::Base(_) => Err(BeaconStateError::IncorrectStateVariant), BeaconState::Altair(state) => Ok(&mut state.previous_epoch_participation), - BeaconState::Merge(state) => Ok(&mut state.previous_epoch_participation), + BeaconState::Bellatrix(state) => Ok(&mut state.previous_epoch_participation), BeaconState::Capella(state) => Ok(&mut state.previous_epoch_participation), BeaconState::Deneb(state) => Ok(&mut state.previous_epoch_participation), BeaconState::Electra(state) => Ok(&mut state.previous_epoch_participation), @@ -1924,7 +1924,7 @@ impl BeaconState { any_pending_mutations |= self_field.has_pending_updates(); }); } - Self::Merge(self_inner) => { + Self::Bellatrix(self_inner) => { map_beacon_state_bellatrix_tree_list_fields_immutable!( self_inner, |_, self_field| { @@ -2052,14 +2052,14 @@ impl BeaconState { ); } (Self::Altair(_), _) => (), - (Self::Merge(self_inner), Self::Merge(base_inner)) => { + (Self::Bellatrix(self_inner), Self::Bellatrix(base_inner)) => { bimap_beacon_state_bellatrix_tree_list_fields!( self_inner, base_inner, |_, self_field, base_field| { self_field.rebase_on(base_field) } ); } - (Self::Merge(_), _) => (), + (Self::Bellatrix(_), _) => (), (Self::Capella(self_inner), Self::Capella(base_inner)) => { bimap_beacon_state_capella_tree_list_fields!( self_inner, @@ -2150,7 +2150,7 @@ impl BeaconState { /// /// We assume this value is stable across forks. This assumption is checked in the /// `check_num_fields_pow2` test. - pub const NUM_FIELDS_POW2: usize = BeaconStateMerge::::NUM_FIELDS.next_power_of_two(); + pub const NUM_FIELDS_POW2: usize = BeaconStateBellatrix::::NUM_FIELDS.next_power_of_two(); /// Specialised deserialisation method that uses the `ChainSpec` as context. #[allow(clippy::arithmetic_side_effects)] @@ -2185,7 +2185,7 @@ impl BeaconState { Self::Altair(inner) => { map_beacon_state_altair_tree_list_fields!(inner, |_, x| { x.apply_updates() }) } - Self::Merge(inner) => { + Self::Bellatrix(inner) => { map_beacon_state_bellatrix_tree_list_fields!(inner, |_, x| { x.apply_updates() }) } Self::Capella(inner) => { @@ -2241,7 +2241,7 @@ impl BeaconState { leaves.push(field.tree_hash_root()); }); } - BeaconState::Merge(state) => { + BeaconState::Bellatrix(state) => { map_beacon_state_bellatrix_fields!(state, |_, field| { leaves.push(field.tree_hash_root()); }); @@ -2331,7 +2331,7 @@ impl CompareFields for BeaconState { match (self, other) { (BeaconState::Base(x), BeaconState::Base(y)) => x.compare_fields(y), (BeaconState::Altair(x), BeaconState::Altair(y)) => x.compare_fields(y), - (BeaconState::Merge(x), BeaconState::Merge(y)) => x.compare_fields(y), + (BeaconState::Bellatrix(x), BeaconState::Bellatrix(y)) => x.compare_fields(y), (BeaconState::Capella(x), BeaconState::Capella(y)) => x.compare_fields(y), (BeaconState::Deneb(x), BeaconState::Deneb(y)) => x.compare_fields(y), (BeaconState::Electra(x), BeaconState::Electra(y)) => x.compare_fields(y), diff --git a/consensus/types/src/beacon_state/progressive_balances_cache.rs b/consensus/types/src/beacon_state/progressive_balances_cache.rs index 523c94cf57e..fd5e51313f7 100644 --- a/consensus/types/src/beacon_state/progressive_balances_cache.rs +++ b/consensus/types/src/beacon_state/progressive_balances_cache.rs @@ -288,7 +288,7 @@ pub fn is_progressive_balances_enabled(state: &BeaconState) -> bo match state { BeaconState::Base(_) => false, BeaconState::Altair(_) - | BeaconState::Merge(_) + | BeaconState::Bellatrix(_) | BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) => true, diff --git a/consensus/types/src/beacon_state/tests.rs b/consensus/types/src/beacon_state/tests.rs index 012c063afef..35afe314957 100644 --- a/consensus/types/src/beacon_state/tests.rs +++ b/consensus/types/src/beacon_state/tests.rs @@ -2,8 +2,8 @@ use crate::{test_utils::*, ForkName}; use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType}; use beacon_chain::types::{ - test_utils::TestRandom, BeaconState, BeaconStateAltair, BeaconStateBase, BeaconStateCapella, - BeaconStateDeneb, BeaconStateElectra, BeaconStateError, BeaconStateMerge, ChainSpec, Domain, + test_utils::TestRandom, BeaconState, BeaconStateAltair, BeaconStateBase, BeaconStateBellatrix, + BeaconStateCapella, BeaconStateDeneb, BeaconStateElectra, BeaconStateError, ChainSpec, Domain, Epoch, EthSpec, Hash256, Keypair, MainnetEthSpec, MinimalEthSpec, RelativeEpoch, Slot, Vector, }; use ssz::Encode; @@ -413,7 +413,7 @@ fn check_num_fields_pow2() { let num_fields = match fork_name { ForkName::Base => BeaconStateBase::::NUM_FIELDS, ForkName::Altair => BeaconStateAltair::::NUM_FIELDS, - ForkName::Merge => BeaconStateMerge::::NUM_FIELDS, + ForkName::Bellatrix => BeaconStateBellatrix::::NUM_FIELDS, ForkName::Capella => BeaconStateCapella::::NUM_FIELDS, ForkName::Deneb => BeaconStateDeneb::::NUM_FIELDS, ForkName::Electra => BeaconStateElectra::::NUM_FIELDS, diff --git a/consensus/types/src/builder_bid.rs b/consensus/types/src/builder_bid.rs index 121d3f84277..9885f78474f 100644 --- a/consensus/types/src/builder_bid.rs +++ b/consensus/types/src/builder_bid.rs @@ -1,7 +1,7 @@ use crate::beacon_block_body::KzgCommitments; use crate::{ - ChainSpec, EthSpec, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderDeneb, - ExecutionPayloadHeaderElectra, ExecutionPayloadHeaderMerge, ExecutionPayloadHeaderRef, + ChainSpec, EthSpec, ExecutionPayloadHeaderBellatrix, ExecutionPayloadHeaderCapella, + ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderElectra, ExecutionPayloadHeaderRef, ExecutionPayloadHeaderRefMut, ForkName, ForkVersionDeserialize, SignedRoot, Uint256, }; use bls::PublicKeyBytes; @@ -11,7 +11,7 @@ use superstruct::superstruct; use tree_hash_derive::TreeHash; #[superstruct( - variants(Merge, Capella, Deneb, Electra), + variants(Bellatrix, Capella, Deneb, Electra), variant_attributes( derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone), serde(bound = "E: EthSpec", deny_unknown_fields) @@ -23,8 +23,8 @@ use tree_hash_derive::TreeHash; #[serde(bound = "E: EthSpec", deny_unknown_fields, untagged)] #[tree_hash(enum_behaviour = "transparent")] pub struct BuilderBid { - #[superstruct(only(Merge), partial_getter(rename = "header_merge"))] - pub header: ExecutionPayloadHeaderMerge, + #[superstruct(only(Bellatrix), partial_getter(rename = "header_bellatrix"))] + pub header: ExecutionPayloadHeaderBellatrix, #[superstruct(only(Capella), partial_getter(rename = "header_capella"))] pub header: ExecutionPayloadHeaderCapella, #[superstruct(only(Deneb), partial_getter(rename = "header_deneb"))] @@ -79,7 +79,9 @@ impl ForkVersionDeserialize for BuilderBid { |e| serde::de::Error::custom(format!("BuilderBid failed to deserialize: {:?}", e)); Ok(match fork_name { - ForkName::Merge => Self::Merge(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Bellatrix => { + Self::Bellatrix(serde_json::from_value(value).map_err(convert_err)?) + } ForkName::Capella => Self::Capella(serde_json::from_value(value).map_err(convert_err)?), ForkName::Deneb => Self::Deneb(serde_json::from_value(value).map_err(convert_err)?), ForkName::Electra => Self::Electra(serde_json::from_value(value).map_err(convert_err)?), diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index c55fe77a1b9..b0346a14ef8 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -147,13 +147,13 @@ pub struct ChainSpec { pub altair_fork_epoch: Option, /* - * Merge hard fork params + * Bellatrix hard fork params */ pub inactivity_penalty_quotient_bellatrix: u64, pub min_slashing_penalty_quotient_bellatrix: u64, pub proportional_slashing_multiplier_bellatrix: u64, pub bellatrix_fork_version: [u8; 4], - /// The Merge fork epoch is optional, with `None` representing "Merge never happens". + /// The Bellatrix fork epoch is optional, with `None` representing "Bellatrix never happens". pub bellatrix_fork_epoch: Option, pub terminal_total_difficulty: Uint256, pub terminal_block_hash: ExecutionBlockHash, @@ -310,7 +310,7 @@ impl ChainSpec { _ => match self.capella_fork_epoch { Some(fork_epoch) if epoch >= fork_epoch => ForkName::Capella, _ => match self.bellatrix_fork_epoch { - Some(fork_epoch) if epoch >= fork_epoch => ForkName::Merge, + Some(fork_epoch) if epoch >= fork_epoch => ForkName::Bellatrix, _ => match self.altair_fork_epoch { Some(fork_epoch) if epoch >= fork_epoch => ForkName::Altair, _ => ForkName::Base, @@ -326,7 +326,7 @@ impl ChainSpec { match fork_name { ForkName::Base => self.genesis_fork_version, ForkName::Altair => self.altair_fork_version, - ForkName::Merge => self.bellatrix_fork_version, + ForkName::Bellatrix => self.bellatrix_fork_version, ForkName::Capella => self.capella_fork_version, ForkName::Deneb => self.deneb_fork_version, ForkName::Electra => self.electra_fork_version, @@ -338,7 +338,7 @@ impl ChainSpec { match fork_name { ForkName::Base => Some(Epoch::new(0)), ForkName::Altair => self.altair_fork_epoch, - ForkName::Merge => self.bellatrix_fork_epoch, + ForkName::Bellatrix => self.bellatrix_fork_epoch, ForkName::Capella => self.capella_fork_epoch, ForkName::Deneb => self.deneb_fork_epoch, ForkName::Electra => self.electra_fork_epoch, @@ -346,7 +346,7 @@ impl ChainSpec { } pub fn inactivity_penalty_quotient_for_fork(&self, fork_name: ForkName) -> u64 { - if fork_name >= ForkName::Merge { + if fork_name >= ForkName::Bellatrix { self.inactivity_penalty_quotient_bellatrix } else if fork_name >= ForkName::Altair { self.inactivity_penalty_quotient_altair @@ -361,7 +361,7 @@ impl ChainSpec { state: &BeaconState, ) -> u64 { let fork_name = state.fork_name_unchecked(); - if fork_name >= ForkName::Merge { + if fork_name >= ForkName::Bellatrix { self.proportional_slashing_multiplier_bellatrix } else if fork_name >= ForkName::Altair { self.proportional_slashing_multiplier_altair @@ -378,7 +378,7 @@ impl ChainSpec { let fork_name = state.fork_name_unchecked(); if fork_name >= ForkName::Electra { self.min_slashing_penalty_quotient_electra - } else if fork_name >= ForkName::Merge { + } else if fork_name >= ForkName::Bellatrix { self.min_slashing_penalty_quotient_bellatrix } else if fork_name >= ForkName::Altair { self.min_slashing_penalty_quotient_altair @@ -693,7 +693,7 @@ impl ChainSpec { altair_fork_epoch: Some(Epoch::new(74240)), /* - * Merge hard fork params + * Bellatrix hard fork params */ inactivity_penalty_quotient_bellatrix: u64::checked_pow(2, 24) .expect("pow does not overflow"), @@ -830,7 +830,7 @@ impl ChainSpec { epochs_per_sync_committee_period: Epoch::new(8), altair_fork_version: [0x01, 0x00, 0x00, 0x01], altair_fork_epoch: None, - // Merge + // Bellatrix bellatrix_fork_version: [0x02, 0x00, 0x00, 0x01], bellatrix_fork_epoch: None, terminal_total_difficulty: Uint256::MAX @@ -993,7 +993,7 @@ impl ChainSpec { altair_fork_epoch: Some(Epoch::new(512)), /* - * Merge hard fork params + * Bellatrix hard fork params */ inactivity_penalty_quotient_bellatrix: u64::checked_pow(2, 24) .expect("pow does not overflow"), diff --git a/consensus/types/src/consts.rs b/consensus/types/src/consts.rs index 31d4c5eac59..c20d5fe8f33 100644 --- a/consensus/types/src/consts.rs +++ b/consensus/types/src/consts.rs @@ -19,7 +19,7 @@ pub mod altair { pub const NUM_FLAG_INDICES: usize = 3; } -pub mod merge { +pub mod bellatrix { pub const INTERVALS_PER_SLOT: u64 = 3; } pub mod deneb { diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index 6749a3f3a2b..62f7f1b8698 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -91,7 +91,7 @@ pub trait EthSpec: /// The number of `sync_committee` subnets. type SyncCommitteeSubnetCount: Unsigned + Clone + Sync + Send + Debug + PartialEq; /* - * New in Merge + * New in Bellatrix */ type MaxBytesPerTransaction: Unsigned + Clone + Sync + Send + Debug + PartialEq; type MaxTransactionsPerPayload: Unsigned + Clone + Sync + Send + Debug + PartialEq; diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index 27dc8cab0a4..a46e7c29fff 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -15,7 +15,7 @@ pub type Transactions = VariableList< pub type Withdrawals = VariableList::MaxWithdrawalsPerPayload>; #[superstruct( - variants(Merge, Capella, Deneb, Electra), + variants(Bellatrix, Capella, Deneb, Electra), variant_attributes( derive( Default, @@ -107,7 +107,9 @@ impl ExecutionPayload { ForkName::Base | ForkName::Altair => Err(ssz::DecodeError::BytesInvalid(format!( "unsupported fork for ExecutionPayload: {fork_name}", ))), - ForkName::Merge => ExecutionPayloadMerge::from_ssz_bytes(bytes).map(Self::Merge), + ForkName::Bellatrix => { + ExecutionPayloadBellatrix::from_ssz_bytes(bytes).map(Self::Bellatrix) + } ForkName::Capella => ExecutionPayloadCapella::from_ssz_bytes(bytes).map(Self::Capella), ForkName::Deneb => ExecutionPayloadDeneb::from_ssz_bytes(bytes).map(Self::Deneb), ForkName::Electra => ExecutionPayloadElectra::from_ssz_bytes(bytes).map(Self::Electra), @@ -116,9 +118,9 @@ impl ExecutionPayload { #[allow(clippy::arithmetic_side_effects)] /// Returns the maximum size of an execution payload. - pub fn max_execution_payload_merge_size() -> usize { + pub fn max_execution_payload_bellatrix_size() -> usize { // Fixed part - ExecutionPayloadMerge::::default().as_ssz_bytes().len() + ExecutionPayloadBellatrix::::default().as_ssz_bytes().len() // Max size of variable length `extra_data` field + (E::max_extra_data_bytes() * ::ssz_fixed_len()) // Max size of variable length `transactions` field @@ -175,7 +177,9 @@ impl ForkVersionDeserialize for ExecutionPayload { }; Ok(match fork_name { - ForkName::Merge => Self::Merge(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Bellatrix => { + Self::Bellatrix(serde_json::from_value(value).map_err(convert_err)?) + } ForkName::Capella => Self::Capella(serde_json::from_value(value).map_err(convert_err)?), ForkName::Deneb => Self::Deneb(serde_json::from_value(value).map_err(convert_err)?), ForkName::Electra => Self::Electra(serde_json::from_value(value).map_err(convert_err)?), @@ -192,7 +196,7 @@ impl ForkVersionDeserialize for ExecutionPayload { impl ExecutionPayload { pub fn fork_name(&self) -> ForkName { match self { - ExecutionPayload::Merge(_) => ForkName::Merge, + ExecutionPayload::Bellatrix(_) => ForkName::Bellatrix, ExecutionPayload::Capella(_) => ForkName::Capella, ExecutionPayload::Deneb(_) => ForkName::Deneb, ExecutionPayload::Electra(_) => ForkName::Electra, diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index 02850304f1d..8515e386c22 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -8,7 +8,7 @@ use tree_hash::TreeHash; use tree_hash_derive::TreeHash; #[superstruct( - variants(Merge, Capella, Deneb, Electra), + variants(Bellatrix, Capella, Deneb, Electra), variant_attributes( derive( Default, @@ -100,7 +100,9 @@ impl ExecutionPayloadHeader { ForkName::Base | ForkName::Altair => Err(ssz::DecodeError::BytesInvalid(format!( "unsupported fork for ExecutionPayloadHeader: {fork_name}", ))), - ForkName::Merge => ExecutionPayloadHeaderMerge::from_ssz_bytes(bytes).map(Self::Merge), + ForkName::Bellatrix => { + ExecutionPayloadHeaderBellatrix::from_ssz_bytes(bytes).map(Self::Bellatrix) + } ForkName::Capella => { ExecutionPayloadHeaderCapella::from_ssz_bytes(bytes).map(Self::Capella) } @@ -118,7 +120,7 @@ impl ExecutionPayloadHeader { match fork_name { ForkName::Base | ForkName::Altair - | ForkName::Merge + | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb | ForkName::Electra => { @@ -138,7 +140,7 @@ impl<'a, E: EthSpec> ExecutionPayloadHeaderRef<'a, E> { } } -impl ExecutionPayloadHeaderMerge { +impl ExecutionPayloadHeaderBellatrix { pub fn upgrade_to_capella(&self) -> ExecutionPayloadHeaderCapella { ExecutionPayloadHeaderCapella { parent_hash: self.parent_hash, @@ -208,8 +210,8 @@ impl ExecutionPayloadHeaderDeneb { } } -impl<'a, E: EthSpec> From<&'a ExecutionPayloadMerge> for ExecutionPayloadHeaderMerge { - fn from(payload: &'a ExecutionPayloadMerge) -> Self { +impl<'a, E: EthSpec> From<&'a ExecutionPayloadBellatrix> for ExecutionPayloadHeaderBellatrix { + fn from(payload: &'a ExecutionPayloadBellatrix) -> Self { Self { parent_hash: payload.parent_hash, fee_recipient: payload.fee_recipient, @@ -301,7 +303,7 @@ impl<'a, E: EthSpec> From<&'a ExecutionPayloadElectra> for ExecutionPayloadHe // These impls are required to work around an inelegance in `to_execution_payload_header`. // They only clone headers so they should be relatively cheap. -impl<'a, E: EthSpec> From<&'a Self> for ExecutionPayloadHeaderMerge { +impl<'a, E: EthSpec> From<&'a Self> for ExecutionPayloadHeaderBellatrix { fn from(payload: &'a Self) -> Self { payload.clone() } @@ -335,11 +337,13 @@ impl<'a, E: EthSpec> From> for ExecutionPayloadHeader } } -impl TryFrom> for ExecutionPayloadHeaderMerge { +impl TryFrom> for ExecutionPayloadHeaderBellatrix { type Error = BeaconStateError; fn try_from(header: ExecutionPayloadHeader) -> Result { match header { - ExecutionPayloadHeader::Merge(execution_payload_header) => Ok(execution_payload_header), + ExecutionPayloadHeader::Bellatrix(execution_payload_header) => { + Ok(execution_payload_header) + } _ => Err(BeaconStateError::IncorrectStateVariant), } } @@ -369,7 +373,7 @@ impl<'a, E: EthSpec> ExecutionPayloadHeaderRefMut<'a, E> { /// Mutate through pub fn replace(self, header: ExecutionPayloadHeader) -> Result<(), BeaconStateError> { match self { - ExecutionPayloadHeaderRefMut::Merge(mut_ref) => { + ExecutionPayloadHeaderRefMut::Bellatrix(mut_ref) => { *mut_ref = header.try_into()?; } ExecutionPayloadHeaderRefMut::Capella(mut_ref) => { @@ -411,7 +415,9 @@ impl ForkVersionDeserialize for ExecutionPayloadHeader { }; Ok(match fork_name { - ForkName::Merge => Self::Merge(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Bellatrix => { + Self::Bellatrix(serde_json::from_value(value).map_err(convert_err)?) + } ForkName::Capella => Self::Capella(serde_json::from_value(value).map_err(convert_err)?), ForkName::Deneb => Self::Deneb(serde_json::from_value(value).map_err(convert_err)?), ForkName::Electra => Self::Electra(serde_json::from_value(value).map_err(convert_err)?), diff --git a/consensus/types/src/fork_context.rs b/consensus/types/src/fork_context.rs index 6b052d83976..0f7f0eb769e 100644 --- a/consensus/types/src/fork_context.rs +++ b/consensus/types/src/fork_context.rs @@ -36,11 +36,11 @@ impl ForkContext { )); } - // Only add Merge to list of forks if it's enabled - // Note: `bellatrix_fork_epoch == None` implies merge hasn't been activated yet on the config. + // Only add Bellatrix to list of forks if it's enabled + // Note: `bellatrix_fork_epoch == None` implies bellatrix hasn't been activated yet on the config. if spec.bellatrix_fork_epoch.is_some() { fork_to_digest.push(( - ForkName::Merge, + ForkName::Bellatrix, ChainSpec::compute_fork_digest( spec.bellatrix_fork_version, genesis_validators_root, diff --git a/consensus/types/src/fork_name.rs b/consensus/types/src/fork_name.rs index f6bd5cbadf9..5cc66214733 100644 --- a/consensus/types/src/fork_name.rs +++ b/consensus/types/src/fork_name.rs @@ -13,7 +13,7 @@ use std::str::FromStr; pub enum ForkName { Base, Altair, - Merge, + Bellatrix, Capella, Deneb, Electra, @@ -24,7 +24,7 @@ impl ForkName { vec![ ForkName::Base, ForkName::Altair, - ForkName::Merge, + ForkName::Bellatrix, ForkName::Capella, ForkName::Deneb, ForkName::Electra, @@ -57,7 +57,7 @@ impl ForkName { spec.electra_fork_epoch = None; spec } - ForkName::Merge => { + ForkName::Bellatrix => { spec.altair_fork_epoch = Some(Epoch::new(0)); spec.bellatrix_fork_epoch = Some(Epoch::new(0)); spec.capella_fork_epoch = None; @@ -99,8 +99,8 @@ impl ForkName { match self { ForkName::Base => None, ForkName::Altair => Some(ForkName::Base), - ForkName::Merge => Some(ForkName::Altair), - ForkName::Capella => Some(ForkName::Merge), + ForkName::Bellatrix => Some(ForkName::Altair), + ForkName::Capella => Some(ForkName::Bellatrix), ForkName::Deneb => Some(ForkName::Capella), ForkName::Electra => Some(ForkName::Deneb), } @@ -112,8 +112,8 @@ impl ForkName { pub fn next_fork(self) -> Option { match self { ForkName::Base => Some(ForkName::Altair), - ForkName::Altair => Some(ForkName::Merge), - ForkName::Merge => Some(ForkName::Capella), + ForkName::Altair => Some(ForkName::Bellatrix), + ForkName::Bellatrix => Some(ForkName::Capella), ForkName::Capella => Some(ForkName::Deneb), ForkName::Deneb => Some(ForkName::Electra), ForkName::Electra => None, @@ -154,9 +154,9 @@ macro_rules! map_fork_name_with { let (value, extra_data) = $body; ($t::Altair(value), extra_data) } - ForkName::Merge => { + ForkName::Bellatrix => { let (value, extra_data) = $body; - ($t::Merge(value), extra_data) + ($t::Bellatrix(value), extra_data) } ForkName::Capella => { let (value, extra_data) = $body; @@ -181,7 +181,7 @@ impl FromStr for ForkName { Ok(match fork_name.to_lowercase().as_ref() { "phase0" | "base" => ForkName::Base, "altair" => ForkName::Altair, - "bellatrix" | "merge" => ForkName::Merge, + "bellatrix" | "merge" => ForkName::Bellatrix, "capella" => ForkName::Capella, "deneb" => ForkName::Deneb, "electra" => ForkName::Electra, @@ -195,7 +195,7 @@ impl Display for ForkName { match self { ForkName::Base => "phase0".fmt(f), ForkName::Altair => "altair".fmt(f), - ForkName::Merge => "bellatrix".fmt(f), + ForkName::Bellatrix => "bellatrix".fmt(f), ForkName::Capella => "capella".fmt(f), ForkName::Deneb => "deneb".fmt(f), ForkName::Electra => "electra".fmt(f), @@ -259,9 +259,9 @@ mod test { #[test] fn fork_name_bellatrix_or_merge() { - assert_eq!(ForkName::from_str("bellatrix"), Ok(ForkName::Merge)); - assert_eq!(ForkName::from_str("merge"), Ok(ForkName::Merge)); - assert_eq!(ForkName::Merge.to_string(), "bellatrix"); + assert_eq!(ForkName::from_str("bellatrix"), Ok(ForkName::Bellatrix)); + assert_eq!(ForkName::from_str("merge"), Ok(ForkName::Bellatrix)); + assert_eq!(ForkName::Bellatrix.to_string(), "bellatrix"); } #[test] diff --git a/consensus/types/src/fork_versioned_response.rs b/consensus/types/src/fork_versioned_response.rs index 195c083e295..cd78b5b3ca0 100644 --- a/consensus/types/src/fork_versioned_response.rs +++ b/consensus/types/src/fork_versioned_response.rs @@ -104,7 +104,8 @@ impl ForkVersionedResponse { #[cfg(test)] mod fork_version_response_tests { use crate::{ - ExecutionPayload, ExecutionPayloadMerge, ForkName, ForkVersionedResponse, MainnetEthSpec, + ExecutionPayload, ExecutionPayloadBellatrix, ForkName, ForkVersionedResponse, + MainnetEthSpec, }; use serde_json::json; @@ -114,9 +115,9 @@ mod fork_version_response_tests { let response_json = serde_json::to_string(&json!(ForkVersionedResponse::> { - version: Some(ForkName::Merge), + version: Some(ForkName::Bellatrix), metadata: Default::default(), - data: ExecutionPayload::Merge(ExecutionPayloadMerge::default()), + data: ExecutionPayload::Bellatrix(ExecutionPayloadBellatrix::default()), })) .unwrap(); @@ -134,7 +135,7 @@ mod fork_version_response_tests { serde_json::to_string(&json!(ForkVersionedResponse::> { version: Some(ForkName::Capella), metadata: Default::default(), - data: ExecutionPayload::Merge(ExecutionPayloadMerge::default()), + data: ExecutionPayload::Bellatrix(ExecutionPayloadBellatrix::default()), })) .unwrap(); diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index feefbc48946..5c521d98af9 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -120,13 +120,13 @@ pub use crate::attestation_data::AttestationData; pub use crate::attestation_duty::AttestationDuty; pub use crate::attester_slashing::AttesterSlashing; pub use crate::beacon_block::{ - BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockCapella, BeaconBlockDeneb, - BeaconBlockElectra, BeaconBlockMerge, BeaconBlockRef, BeaconBlockRefMut, BlindedBeaconBlock, + BeaconBlock, BeaconBlockAltair, BeaconBlockBase, BeaconBlockBellatrix, BeaconBlockCapella, + BeaconBlockDeneb, BeaconBlockElectra, BeaconBlockRef, BeaconBlockRefMut, BlindedBeaconBlock, EmptyBlock, }; pub use crate::beacon_block_body::{ - BeaconBlockBody, BeaconBlockBodyAltair, BeaconBlockBodyBase, BeaconBlockBodyCapella, - BeaconBlockBodyDeneb, BeaconBlockBodyElectra, BeaconBlockBodyMerge, BeaconBlockBodyRef, + BeaconBlockBody, BeaconBlockBodyAltair, BeaconBlockBodyBase, BeaconBlockBodyBellatrix, + BeaconBlockBodyCapella, BeaconBlockBodyDeneb, BeaconBlockBodyElectra, BeaconBlockBodyRef, BeaconBlockBodyRefMut, }; pub use crate::beacon_block_header::BeaconBlockHeader; @@ -154,12 +154,12 @@ pub use crate::execution_block_hash::ExecutionBlockHash; pub use crate::execution_block_header::ExecutionBlockHeader; pub use crate::execution_layer_withdrawal_request::ExecutionLayerWithdrawalRequest; pub use crate::execution_payload::{ - ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadElectra, - ExecutionPayloadMerge, ExecutionPayloadRef, Transaction, Transactions, Withdrawals, + ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella, ExecutionPayloadDeneb, + ExecutionPayloadElectra, ExecutionPayloadRef, Transaction, Transactions, Withdrawals, }; pub use crate::execution_payload_header::{ - ExecutionPayloadHeader, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderDeneb, - ExecutionPayloadHeaderElectra, ExecutionPayloadHeaderMerge, ExecutionPayloadHeaderRef, + ExecutionPayloadHeader, ExecutionPayloadHeaderBellatrix, ExecutionPayloadHeaderCapella, + ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderElectra, ExecutionPayloadHeaderRef, ExecutionPayloadHeaderRefMut, }; pub use crate::fork::Fork; @@ -192,9 +192,9 @@ pub use crate::light_client_update::{ pub use crate::participation_flags::ParticipationFlags; pub use crate::participation_list::ParticipationList; pub use crate::payload::{ - AbstractExecPayload, BlindedPayload, BlindedPayloadCapella, BlindedPayloadDeneb, - BlindedPayloadElectra, BlindedPayloadMerge, BlindedPayloadRef, BlockType, ExecPayload, - FullPayload, FullPayloadCapella, FullPayloadDeneb, FullPayloadElectra, FullPayloadMerge, + AbstractExecPayload, BlindedPayload, BlindedPayloadBellatrix, BlindedPayloadCapella, + BlindedPayloadDeneb, BlindedPayloadElectra, BlindedPayloadRef, BlockType, ExecPayload, + FullPayload, FullPayloadBellatrix, FullPayloadCapella, FullPayloadDeneb, FullPayloadElectra, FullPayloadRef, OwnedExecPayload, }; pub use crate::pending_attestation::PendingAttestation; @@ -213,9 +213,9 @@ pub use crate::shuffling_id::AttestationShufflingId; pub use crate::signed_aggregate_and_proof::SignedAggregateAndProof; pub use crate::signed_beacon_block::{ ssz_tagged_signed_beacon_block, ssz_tagged_signed_beacon_block_arc, SignedBeaconBlock, - SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockCapella, - SignedBeaconBlockDeneb, SignedBeaconBlockElectra, SignedBeaconBlockHash, - SignedBeaconBlockMerge, SignedBlindedBeaconBlock, + SignedBeaconBlockAltair, SignedBeaconBlockBase, SignedBeaconBlockBellatrix, + SignedBeaconBlockCapella, SignedBeaconBlockDeneb, SignedBeaconBlockElectra, + SignedBeaconBlockHash, SignedBlindedBeaconBlock, }; pub use crate::signed_beacon_block_header::SignedBeaconBlockHeader; pub use crate::signed_bls_to_execution_change::SignedBlsToExecutionChange; diff --git a/consensus/types/src/light_client_bootstrap.rs b/consensus/types/src/light_client_bootstrap.rs index 4d42d357c19..61da0e1b117 100644 --- a/consensus/types/src/light_client_bootstrap.rs +++ b/consensus/types/src/light_client_bootstrap.rs @@ -78,7 +78,7 @@ impl LightClientBootstrap { pub fn from_ssz_bytes(bytes: &[u8], fork_name: ForkName) -> Result { let bootstrap = match fork_name { - ForkName::Altair | ForkName::Merge => { + ForkName::Altair | ForkName::Bellatrix => { Self::Altair(LightClientBootstrapAltair::from_ssz_bytes(bytes)?) } ForkName::Capella => Self::Capella(LightClientBootstrapCapella::from_ssz_bytes(bytes)?), @@ -101,7 +101,7 @@ impl LightClientBootstrap { match fork_name { ForkName::Base => 0, ForkName::Altair - | ForkName::Merge + | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb | ForkName::Electra => { @@ -128,7 +128,7 @@ impl LightClientBootstrap { .map_err(|_| Error::InconsistentFork)? { ForkName::Base => return Err(Error::AltairForkNotActive), - ForkName::Altair | ForkName::Merge => Self::Altair(LightClientBootstrapAltair { + ForkName::Altair | ForkName::Bellatrix => Self::Altair(LightClientBootstrapAltair { header: LightClientHeaderAltair::block_to_light_client_header(block)?, current_sync_committee, current_sync_committee_branch, diff --git a/consensus/types/src/light_client_finality_update.rs b/consensus/types/src/light_client_finality_update.rs index 5f83b0db523..29c526e2916 100644 --- a/consensus/types/src/light_client_finality_update.rs +++ b/consensus/types/src/light_client_finality_update.rs @@ -79,7 +79,7 @@ impl LightClientFinalityUpdate { .fork_name(chain_spec) .map_err(|_| Error::InconsistentFork)? { - ForkName::Altair | ForkName::Merge => { + ForkName::Altair | ForkName::Bellatrix => { let finality_update = LightClientFinalityUpdateAltair { attested_header: LightClientHeaderAltair::block_to_light_client_header( attested_block, @@ -147,7 +147,7 @@ impl LightClientFinalityUpdate { pub fn from_ssz_bytes(bytes: &[u8], fork_name: ForkName) -> Result { let finality_update = match fork_name { - ForkName::Altair | ForkName::Merge => { + ForkName::Altair | ForkName::Bellatrix => { Self::Altair(LightClientFinalityUpdateAltair::from_ssz_bytes(bytes)?) } ForkName::Capella => { @@ -172,7 +172,7 @@ impl LightClientFinalityUpdate { match fork_name { ForkName::Base => 0, ForkName::Altair - | ForkName::Merge + | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb | ForkName::Electra => { diff --git a/consensus/types/src/light_client_header.rs b/consensus/types/src/light_client_header.rs index 647ece99499..213ec90f955 100644 --- a/consensus/types/src/light_client_header.rs +++ b/consensus/types/src/light_client_header.rs @@ -75,7 +75,7 @@ impl LightClientHeader { .map_err(|_| Error::InconsistentFork)? { ForkName::Base => return Err(Error::AltairForkNotActive), - ForkName::Altair | ForkName::Merge => LightClientHeader::Altair( + ForkName::Altair | ForkName::Bellatrix => LightClientHeader::Altair( LightClientHeaderAltair::block_to_light_client_header(block)?, ), ForkName::Capella => LightClientHeader::Capella( @@ -90,7 +90,7 @@ impl LightClientHeader { pub fn from_ssz_bytes(bytes: &[u8], fork_name: ForkName) -> Result { let header = match fork_name { - ForkName::Altair | ForkName::Merge => { + ForkName::Altair | ForkName::Bellatrix => { LightClientHeader::Altair(LightClientHeaderAltair::from_ssz_bytes(bytes)?) } ForkName::Capella => { @@ -119,7 +119,7 @@ impl LightClientHeader { pub fn ssz_max_var_len_for_fork(fork_name: ForkName) -> usize { match fork_name { - ForkName::Base | ForkName::Altair | ForkName::Merge => 0, + ForkName::Base | ForkName::Altair | ForkName::Bellatrix => 0, ForkName::Capella | ForkName::Deneb | ForkName::Electra => { ExecutionPayloadHeader::::ssz_max_var_len_for_fork(fork_name) } @@ -198,7 +198,7 @@ impl ForkVersionDeserialize for LightClientHeader { fork_name: ForkName, ) -> Result { match fork_name { - ForkName::Altair | ForkName::Merge => serde_json::from_value(value) + ForkName::Altair | ForkName::Bellatrix => serde_json::from_value(value) .map(|light_client_header| Self::Altair(light_client_header)) .map_err(serde::de::Error::custom), ForkName::Capella => serde_json::from_value(value) diff --git a/consensus/types/src/light_client_optimistic_update.rs b/consensus/types/src/light_client_optimistic_update.rs index d22c4535f1e..4727673f6c0 100644 --- a/consensus/types/src/light_client_optimistic_update.rs +++ b/consensus/types/src/light_client_optimistic_update.rs @@ -70,13 +70,15 @@ impl LightClientOptimisticUpdate { .fork_name(chain_spec) .map_err(|_| Error::InconsistentFork)? { - ForkName::Altair | ForkName::Merge => Self::Altair(LightClientOptimisticUpdateAltair { - attested_header: LightClientHeaderAltair::block_to_light_client_header( - attested_block, - )?, - sync_aggregate, - signature_slot, - }), + ForkName::Altair | ForkName::Bellatrix => { + Self::Altair(LightClientOptimisticUpdateAltair { + attested_header: LightClientHeaderAltair::block_to_light_client_header( + attested_block, + )?, + sync_aggregate, + signature_slot, + }) + } ForkName::Capella => Self::Capella(LightClientOptimisticUpdateCapella { attested_header: LightClientHeaderCapella::block_to_light_client_header( attested_block, @@ -131,7 +133,7 @@ impl LightClientOptimisticUpdate { pub fn from_ssz_bytes(bytes: &[u8], fork_name: ForkName) -> Result { let optimistic_update = match fork_name { - ForkName::Altair | ForkName::Merge => { + ForkName::Altair | ForkName::Bellatrix => { Self::Altair(LightClientOptimisticUpdateAltair::from_ssz_bytes(bytes)?) } ForkName::Capella => { @@ -156,7 +158,7 @@ impl LightClientOptimisticUpdate { match fork_name { ForkName::Base => 0, ForkName::Altair - | ForkName::Merge + | ForkName::Bellatrix | ForkName::Capella | ForkName::Deneb | ForkName::Electra => { diff --git a/consensus/types/src/light_client_update.rs b/consensus/types/src/light_client_update.rs index d5e8cd592df..002fbea2d37 100644 --- a/consensus/types/src/light_client_update.rs +++ b/consensus/types/src/light_client_update.rs @@ -191,7 +191,7 @@ impl LightClientUpdate { .map_err(|_| Error::InconsistentFork)? { ForkName::Base => return Err(Error::AltairForkNotActive), - ForkName::Altair | ForkName::Merge => { + ForkName::Altair | ForkName::Bellatrix => { let attested_header = LightClientHeaderAltair::block_to_light_client_header(attested_block)?; let finalized_header = @@ -243,7 +243,7 @@ impl LightClientUpdate { pub fn from_ssz_bytes(bytes: &[u8], fork_name: ForkName) -> Result { let update = match fork_name { - ForkName::Altair | ForkName::Merge => { + ForkName::Altair | ForkName::Bellatrix => { Self::Altair(LightClientUpdateAltair::from_ssz_bytes(bytes)?) } ForkName::Capella => Self::Capella(LightClientUpdateCapella::from_ssz_bytes(bytes)?), diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index 18b3199bd35..80a70c171f5 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -79,22 +79,22 @@ pub trait AbstractExecPayload: + Sized + From> + TryFrom> - + TryInto + + TryInto + TryInto + TryInto + TryInto { type Ref<'a>: ExecPayload + Copy - + From<&'a Self::Merge> + + From<&'a Self::Bellatrix> + From<&'a Self::Capella> + From<&'a Self::Deneb> + From<&'a Self::Electra>; - type Merge: OwnedExecPayload + type Bellatrix: OwnedExecPayload + Into - + for<'a> From>> - + TryFrom>; + + for<'a> From>> + + TryFrom>; type Capella: OwnedExecPayload + Into + for<'a> From>> @@ -110,7 +110,7 @@ pub trait AbstractExecPayload: } #[superstruct( - variants(Merge, Capella, Deneb, Electra), + variants(Bellatrix, Capella, Deneb, Electra), variant_attributes( derive( Debug, @@ -145,8 +145,11 @@ pub trait AbstractExecPayload: #[arbitrary(bound = "E: EthSpec")] #[tree_hash(enum_behaviour = "transparent")] pub struct FullPayload { - #[superstruct(only(Merge), partial_getter(rename = "execution_payload_merge"))] - pub execution_payload: ExecutionPayloadMerge, + #[superstruct( + only(Bellatrix), + partial_getter(rename = "execution_payload_bellatrix") + )] + pub execution_payload: ExecutionPayloadBellatrix, #[superstruct(only(Capella), partial_getter(rename = "execution_payload_capella"))] pub execution_payload: ExecutionPayloadCapella, #[superstruct(only(Deneb), partial_getter(rename = "execution_payload_deneb"))] @@ -252,7 +255,7 @@ impl ExecPayload for FullPayload { fn withdrawals_root(&self) -> Result { match self { - FullPayload::Merge(_) => Err(Error::IncorrectStateVariant), + FullPayload::Bellatrix(_) => Err(Error::IncorrectStateVariant), FullPayload::Capella(ref inner) => { Ok(inner.execution_payload.withdrawals.tree_hash_root()) } @@ -267,7 +270,9 @@ impl ExecPayload for FullPayload { fn blob_gas_used(&self) -> Result { match self { - FullPayload::Merge(_) | FullPayload::Capella(_) => Err(Error::IncorrectStateVariant), + FullPayload::Bellatrix(_) | FullPayload::Capella(_) => { + Err(Error::IncorrectStateVariant) + } FullPayload::Deneb(ref inner) => Ok(inner.execution_payload.blob_gas_used), FullPayload::Electra(ref inner) => Ok(inner.execution_payload.blob_gas_used), } @@ -296,7 +301,7 @@ impl FullPayload { pub fn default_at_fork(fork_name: ForkName) -> Result { match fork_name { ForkName::Base | ForkName::Altair => Err(Error::IncorrectStateVariant), - ForkName::Merge => Ok(FullPayloadMerge::default().into()), + ForkName::Bellatrix => Ok(FullPayloadBellatrix::default().into()), ForkName::Capella => Ok(FullPayloadCapella::default().into()), ForkName::Deneb => Ok(FullPayloadDeneb::default().into()), ForkName::Electra => Ok(FullPayloadElectra::default().into()), @@ -382,7 +387,7 @@ impl<'b, E: EthSpec> ExecPayload for FullPayloadRef<'b, E> { fn withdrawals_root(&self) -> Result { match self { - FullPayloadRef::Merge(_) => Err(Error::IncorrectStateVariant), + FullPayloadRef::Bellatrix(_) => Err(Error::IncorrectStateVariant), FullPayloadRef::Capella(inner) => { Ok(inner.execution_payload.withdrawals.tree_hash_root()) } @@ -397,7 +402,7 @@ impl<'b, E: EthSpec> ExecPayload for FullPayloadRef<'b, E> { fn blob_gas_used(&self) -> Result { match self { - FullPayloadRef::Merge(_) | FullPayloadRef::Capella(_) => { + FullPayloadRef::Bellatrix(_) | FullPayloadRef::Capella(_) => { Err(Error::IncorrectStateVariant) } FullPayloadRef::Deneb(inner) => Ok(inner.execution_payload.blob_gas_used), @@ -420,7 +425,7 @@ impl<'b, E: EthSpec> ExecPayload for FullPayloadRef<'b, E> { impl AbstractExecPayload for FullPayload { type Ref<'a> = FullPayloadRef<'a, E>; - type Merge = FullPayloadMerge; + type Bellatrix = FullPayloadBellatrix; type Capella = FullPayloadCapella; type Deneb = FullPayloadDeneb; type Electra = FullPayloadElectra; @@ -442,7 +447,7 @@ impl TryFrom> for FullPayload { } #[superstruct( - variants(Merge, Capella, Deneb, Electra), + variants(Bellatrix, Capella, Deneb, Electra), variant_attributes( derive( Debug, @@ -476,8 +481,11 @@ impl TryFrom> for FullPayload { #[arbitrary(bound = "E: EthSpec")] #[tree_hash(enum_behaviour = "transparent")] pub struct BlindedPayload { - #[superstruct(only(Merge), partial_getter(rename = "execution_payload_merge"))] - pub execution_payload_header: ExecutionPayloadHeaderMerge, + #[superstruct( + only(Bellatrix), + partial_getter(rename = "execution_payload_bellatrix") + )] + pub execution_payload_header: ExecutionPayloadHeaderBellatrix, #[superstruct(only(Capella), partial_getter(rename = "execution_payload_capella"))] pub execution_payload_header: ExecutionPayloadHeaderCapella, #[superstruct(only(Deneb), partial_getter(rename = "execution_payload_deneb"))] @@ -561,7 +569,7 @@ impl ExecPayload for BlindedPayload { fn withdrawals_root(&self) -> Result { match self { - BlindedPayload::Merge(_) => Err(Error::IncorrectStateVariant), + BlindedPayload::Bellatrix(_) => Err(Error::IncorrectStateVariant), BlindedPayload::Capella(ref inner) => { Ok(inner.execution_payload_header.withdrawals_root) } @@ -574,7 +582,7 @@ impl ExecPayload for BlindedPayload { fn blob_gas_used(&self) -> Result { match self { - BlindedPayload::Merge(_) | BlindedPayload::Capella(_) => { + BlindedPayload::Bellatrix(_) | BlindedPayload::Capella(_) => { Err(Error::IncorrectStateVariant) } BlindedPayload::Deneb(ref inner) => Ok(inner.execution_payload_header.blob_gas_used), @@ -662,7 +670,7 @@ impl<'b, E: EthSpec> ExecPayload for BlindedPayloadRef<'b, E> { fn withdrawals_root(&self) -> Result { match self { - BlindedPayloadRef::Merge(_) => Err(Error::IncorrectStateVariant), + BlindedPayloadRef::Bellatrix(_) => Err(Error::IncorrectStateVariant), BlindedPayloadRef::Capella(inner) => { Ok(inner.execution_payload_header.withdrawals_root) } @@ -675,7 +683,7 @@ impl<'b, E: EthSpec> ExecPayload for BlindedPayloadRef<'b, E> { fn blob_gas_used(&self) -> Result { match self { - BlindedPayloadRef::Merge(_) | BlindedPayloadRef::Capella(_) => { + BlindedPayloadRef::Bellatrix(_) | BlindedPayloadRef::Capella(_) => { Err(Error::IncorrectStateVariant) } BlindedPayloadRef::Deneb(inner) => Ok(inner.execution_payload_header.blob_gas_used), @@ -699,12 +707,12 @@ impl<'b, E: EthSpec> ExecPayload for BlindedPayloadRef<'b, E> { } macro_rules! impl_exec_payload_common { - ($wrapper_type:ident, // BlindedPayloadMerge | FullPayloadMerge - $wrapped_type:ident, // ExecutionPayloadHeaderMerge | ExecutionPayloadMerge - $wrapped_type_full:ident, // ExecutionPayloadMerge | ExecutionPayloadMerge - $wrapped_type_header:ident, // ExecutionPayloadHeaderMerge | ExecutionPayloadHeaderMerge + ($wrapper_type:ident, // BlindedPayloadBellatrix | FullPayloadBellatrix + $wrapped_type:ident, // ExecutionPayloadHeaderBellatrix | ExecutionPayloadBellatrix + $wrapped_type_full:ident, // ExecutionPayloadBellatrix | ExecutionPayloadBellatrix + $wrapped_type_header:ident, // ExecutionPayloadHeaderBellatrix | ExecutionPayloadHeaderBellatrix $wrapped_field:ident, // execution_payload_header | execution_payload - $fork_variant:ident, // Merge | Merge + $fork_variant:ident, // Bellatrix | Bellatrix $block_type_variant:ident, // Blinded | Full $is_default_with_empty_roots:block, $f:block, @@ -783,17 +791,17 @@ macro_rules! impl_exec_payload_common { } macro_rules! impl_exec_payload_for_fork { - // BlindedPayloadMerge, FullPayloadMerge, ExecutionPayloadHeaderMerge, ExecutionPayloadMerge, Merge + // BlindedPayloadBellatrix, FullPayloadBellatrix, ExecutionPayloadHeaderBellatrix, ExecutionPayloadBellatrix, Bellatrix ($wrapper_type_header:ident, $wrapper_type_full:ident, $wrapped_type_header:ident, $wrapped_type_full:ident, $fork_variant:ident) => { //*************** Blinded payload implementations ******************// impl_exec_payload_common!( - $wrapper_type_header, // BlindedPayloadMerge - $wrapped_type_header, // ExecutionPayloadHeaderMerge - $wrapped_type_full, // ExecutionPayloadMerge - $wrapped_type_header, // ExecutionPayloadHeaderMerge + $wrapper_type_header, // BlindedPayloadBellatrix + $wrapped_type_header, // ExecutionPayloadHeaderBellatrix + $wrapped_type_full, // ExecutionPayloadBellatrix + $wrapped_type_header, // ExecutionPayloadHeaderBellatrix execution_payload_header, - $fork_variant, // Merge + $fork_variant, // Bellatrix Blinded, { |wrapper: &$wrapper_type_header| { @@ -872,12 +880,12 @@ macro_rules! impl_exec_payload_for_fork { //*************** Full payload implementations ******************// impl_exec_payload_common!( - $wrapper_type_full, // FullPayloadMerge - $wrapped_type_full, // ExecutionPayloadMerge - $wrapped_type_full, // ExecutionPayloadMerge - $wrapped_type_header, // ExecutionPayloadHeaderMerge + $wrapper_type_full, // FullPayloadBellatrix + $wrapped_type_full, // ExecutionPayloadBellatrix + $wrapped_type_full, // ExecutionPayloadBellatrix + $wrapped_type_header, // ExecutionPayloadHeaderBellatrix execution_payload, - $fork_variant, // Merge + $fork_variant, // Bellatrix Full, { |wrapper: &$wrapper_type_full| { @@ -952,11 +960,11 @@ macro_rules! impl_exec_payload_for_fork { } impl_exec_payload_for_fork!( - BlindedPayloadMerge, - FullPayloadMerge, - ExecutionPayloadHeaderMerge, - ExecutionPayloadMerge, - Merge + BlindedPayloadBellatrix, + FullPayloadBellatrix, + ExecutionPayloadHeaderBellatrix, + ExecutionPayloadBellatrix, + Bellatrix ); impl_exec_payload_for_fork!( BlindedPayloadCapella, @@ -982,7 +990,7 @@ impl_exec_payload_for_fork!( impl AbstractExecPayload for BlindedPayload { type Ref<'a> = BlindedPayloadRef<'a, E>; - type Merge = BlindedPayloadMerge; + type Bellatrix = BlindedPayloadBellatrix; type Capella = BlindedPayloadCapella; type Deneb = BlindedPayloadDeneb; type Electra = BlindedPayloadElectra; @@ -1002,8 +1010,8 @@ impl From> for BlindedPayload { impl From> for BlindedPayload { fn from(execution_payload_header: ExecutionPayloadHeader) -> Self { match execution_payload_header { - ExecutionPayloadHeader::Merge(execution_payload_header) => { - Self::Merge(BlindedPayloadMerge { + ExecutionPayloadHeader::Bellatrix(execution_payload_header) => { + Self::Bellatrix(BlindedPayloadBellatrix { execution_payload_header, }) } @@ -1029,8 +1037,8 @@ impl From> for BlindedPayload { impl From> for ExecutionPayloadHeader { fn from(blinded: BlindedPayload) -> Self { match blinded { - BlindedPayload::Merge(blinded_payload) => { - ExecutionPayloadHeader::Merge(blinded_payload.execution_payload_header) + BlindedPayload::Bellatrix(blinded_payload) => { + ExecutionPayloadHeader::Bellatrix(blinded_payload.execution_payload_header) } BlindedPayload::Capella(blinded_payload) => { ExecutionPayloadHeader::Capella(blinded_payload.execution_payload_header) diff --git a/consensus/types/src/signed_beacon_block.rs b/consensus/types/src/signed_beacon_block.rs index da4ac392362..4d3279a7f77 100644 --- a/consensus/types/src/signed_beacon_block.rs +++ b/consensus/types/src/signed_beacon_block.rs @@ -37,7 +37,7 @@ impl From for Hash256 { /// A `BeaconBlock` and a signature from its proposer. #[superstruct( - variants(Base, Altair, Merge, Capella, Deneb, Electra), + variants(Base, Altair, Bellatrix, Capella, Deneb, Electra), variant_attributes( derive( Debug, @@ -72,8 +72,8 @@ pub struct SignedBeaconBlock = FullP pub message: BeaconBlockBase, #[superstruct(only(Altair), partial_getter(rename = "message_altair"))] pub message: BeaconBlockAltair, - #[superstruct(only(Merge), partial_getter(rename = "message_merge"))] - pub message: BeaconBlockMerge, + #[superstruct(only(Bellatrix), partial_getter(rename = "message_bellatrix"))] + pub message: BeaconBlockBellatrix, #[superstruct(only(Capella), partial_getter(rename = "message_capella"))] pub message: BeaconBlockCapella, #[superstruct(only(Deneb), partial_getter(rename = "message_deneb"))] @@ -150,8 +150,8 @@ impl> SignedBeaconBlock BeaconBlock::Altair(message) => { SignedBeaconBlock::Altair(SignedBeaconBlockAltair { message, signature }) } - BeaconBlock::Merge(message) => { - SignedBeaconBlock::Merge(SignedBeaconBlockMerge { message, signature }) + BeaconBlock::Bellatrix(message) => { + SignedBeaconBlock::Bellatrix(SignedBeaconBlockBellatrix { message, signature }) } BeaconBlock::Capella(message) => { SignedBeaconBlock::Capella(SignedBeaconBlockCapella { message, signature }) @@ -310,20 +310,20 @@ impl From>> // Post-Bellatrix blocks can be "unblinded" by adding the full payload. // NOTE: It might be nice to come up with a `superstruct` pattern to abstract over this before // the first fork after Bellatrix. -impl SignedBeaconBlockMerge> { +impl SignedBeaconBlockBellatrix> { pub fn into_full_block( self, - execution_payload: ExecutionPayloadMerge, - ) -> SignedBeaconBlockMerge> { - let SignedBeaconBlockMerge { + execution_payload: ExecutionPayloadBellatrix, + ) -> SignedBeaconBlockBellatrix> { + let SignedBeaconBlockBellatrix { message: - BeaconBlockMerge { + BeaconBlockBellatrix { slot, proposer_index, parent_root, state_root, body: - BeaconBlockBodyMerge { + BeaconBlockBodyBellatrix { randao_reveal, eth1_data, graffiti, @@ -333,18 +333,18 @@ impl SignedBeaconBlockMerge> { deposits, voluntary_exits, sync_aggregate, - execution_payload: BlindedPayloadMerge { .. }, + execution_payload: BlindedPayloadBellatrix { .. }, }, }, signature, } = self; - SignedBeaconBlockMerge { - message: BeaconBlockMerge { + SignedBeaconBlockBellatrix { + message: BeaconBlockBellatrix { slot, proposer_index, parent_root, state_root, - body: BeaconBlockBodyMerge { + body: BeaconBlockBodyBellatrix { randao_reveal, eth1_data, graffiti, @@ -354,7 +354,7 @@ impl SignedBeaconBlockMerge> { deposits, voluntary_exits, sync_aggregate, - execution_payload: FullPayloadMerge { execution_payload }, + execution_payload: FullPayloadBellatrix { execution_payload }, }, }, signature, @@ -536,8 +536,8 @@ impl SignedBeaconBlock> { let full_block = match (self, execution_payload) { (SignedBeaconBlock::Base(block), _) => SignedBeaconBlock::Base(block.into()), (SignedBeaconBlock::Altair(block), _) => SignedBeaconBlock::Altair(block.into()), - (SignedBeaconBlock::Merge(block), Some(ExecutionPayload::Merge(payload))) => { - SignedBeaconBlock::Merge(block.into_full_block(payload)) + (SignedBeaconBlock::Bellatrix(block), Some(ExecutionPayload::Bellatrix(payload))) => { + SignedBeaconBlock::Bellatrix(block.into_full_block(payload)) } (SignedBeaconBlock::Capella(block), Some(ExecutionPayload::Capella(payload))) => { SignedBeaconBlock::Capella(block.into_full_block(payload)) @@ -550,7 +550,7 @@ impl SignedBeaconBlock> { } // avoid wildcard matching forks so that compiler will // direct us here when a new fork has been added - (SignedBeaconBlock::Merge(_), _) => return None, + (SignedBeaconBlock::Bellatrix(_), _) => return None, (SignedBeaconBlock::Capella(_), _) => return None, (SignedBeaconBlock::Deneb(_), _) => return None, (SignedBeaconBlock::Electra(_), _) => return None, @@ -687,8 +687,8 @@ pub mod ssz_tagged_signed_beacon_block { ForkName::Altair => Ok(SignedBeaconBlock::Altair( SignedBeaconBlockAltair::from_ssz_bytes(body)?, )), - ForkName::Merge => Ok(SignedBeaconBlock::Merge( - SignedBeaconBlockMerge::from_ssz_bytes(body)?, + ForkName::Bellatrix => Ok(SignedBeaconBlock::Bellatrix( + SignedBeaconBlockBellatrix::from_ssz_bytes(body)?, )), ForkName::Capella => Ok(SignedBeaconBlock::Capella( SignedBeaconBlockCapella::from_ssz_bytes(body)?, @@ -744,7 +744,10 @@ mod test { BeaconBlock::Altair(BeaconBlockAltair::empty(spec)), sig.clone(), ), - SignedBeaconBlock::from_block(BeaconBlock::Merge(BeaconBlockMerge::empty(spec)), sig), + SignedBeaconBlock::from_block( + BeaconBlock::Bellatrix(BeaconBlockBellatrix::empty(spec)), + sig, + ), ]; for block in blocks { @@ -783,7 +786,7 @@ mod test { sig.clone(), ), SignedBeaconBlock::from_block( - BeaconBlock::Merge(BeaconBlockMerge::empty(spec)), + BeaconBlock::Bellatrix(BeaconBlockBellatrix::empty(spec)), sig.clone(), ), SignedBeaconBlock::from_block( diff --git a/consensus/types/src/voluntary_exit.rs b/consensus/types/src/voluntary_exit.rs index 74175423e34..4c7c16757ed 100644 --- a/consensus/types/src/voluntary_exit.rs +++ b/consensus/types/src/voluntary_exit.rs @@ -42,7 +42,7 @@ impl VoluntaryExit { ) -> SignedVoluntaryExit { let fork_name = spec.fork_name_at_epoch(self.epoch); let fork_version = match fork_name { - ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => { spec.fork_version_for_name(fork_name) } // EIP-7044 diff --git a/lcli/src/create_payload_header.rs b/lcli/src/create_payload_header.rs index 7aa1ef70089..974a34591f0 100644 --- a/lcli/src/create_payload_header.rs +++ b/lcli/src/create_payload_header.rs @@ -5,8 +5,9 @@ use std::fs::File; use std::io::Write; use std::time::{SystemTime, UNIX_EPOCH}; use types::{ - EthSpec, ExecutionPayloadHeader, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderDeneb, - ExecutionPayloadHeaderElectra, ExecutionPayloadHeaderMerge, ForkName, + EthSpec, ExecutionPayloadHeader, ExecutionPayloadHeaderBellatrix, + ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderElectra, + ForkName, }; pub fn run(matches: &ArgMatches) -> Result<(), String> { @@ -20,17 +21,17 @@ pub fn run(matches: &ArgMatches) -> Result<(), String> { let base_fee_per_gas = parse_required(matches, "base-fee-per-gas")?; let gas_limit = parse_required(matches, "gas-limit")?; let file_name = matches.value_of("file").ok_or("No file supplied")?; - let fork_name: ForkName = parse_optional(matches, "fork")?.unwrap_or(ForkName::Merge); + let fork_name: ForkName = parse_optional(matches, "fork")?.unwrap_or(ForkName::Bellatrix); let execution_payload_header: ExecutionPayloadHeader = match fork_name { ForkName::Base | ForkName::Altair => return Err("invalid fork name".to_string()), - ForkName::Merge => ExecutionPayloadHeader::Merge(ExecutionPayloadHeaderMerge { + ForkName::Bellatrix => ExecutionPayloadHeader::Bellatrix(ExecutionPayloadHeaderBellatrix { gas_limit, base_fee_per_gas, timestamp: genesis_time, block_hash: eth1_block_hash, prev_randao: eth1_block_hash.into_root(), - ..ExecutionPayloadHeaderMerge::default() + ..ExecutionPayloadHeaderBellatrix::default() }), ForkName::Capella => ExecutionPayloadHeader::Capella(ExecutionPayloadHeaderCapella { gas_limit, diff --git a/lcli/src/main.rs b/lcli/src/main.rs index c374a8f4b37..7b5c1598c9e 100644 --- a/lcli/src/main.rs +++ b/lcli/src/main.rs @@ -433,7 +433,7 @@ fn main() { .takes_value(true) .default_value("bellatrix") .help("The fork for which the execution payload header should be created.") - .possible_values(&["merge", "bellatrix", "capella", "deneb", "electra"]) + .possible_values(&["bellatrix", "capella", "deneb", "electra"]) ) ) .subcommand( diff --git a/lcli/src/new_testnet.rs b/lcli/src/new_testnet.rs index edba4249966..f6bfb2ac013 100644 --- a/lcli/src/new_testnet.rs +++ b/lcli/src/new_testnet.rs @@ -21,8 +21,8 @@ use std::time::{SystemTime, UNIX_EPOCH}; use types::ExecutionBlockHash; use types::{ test_utils::generate_deterministic_keypairs, Address, BeaconState, ChainSpec, Config, Epoch, - Eth1Data, EthSpec, ExecutionPayloadHeader, ExecutionPayloadHeaderCapella, - ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderElectra, ExecutionPayloadHeaderMerge, + Eth1Data, EthSpec, ExecutionPayloadHeader, ExecutionPayloadHeaderBellatrix, + ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderElectra, ForkName, Hash256, Keypair, PublicKey, Validator, }; @@ -114,9 +114,9 @@ pub fn run(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Resul ForkName::Base | ForkName::Altair => Err(ssz::DecodeError::BytesInvalid( "genesis fork must be post-merge".to_string(), )), - ForkName::Merge => { - ExecutionPayloadHeaderMerge::::from_ssz_bytes(bytes.as_slice()) - .map(ExecutionPayloadHeader::Merge) + ForkName::Bellatrix => { + ExecutionPayloadHeaderBellatrix::::from_ssz_bytes(bytes.as_slice()) + .map(ExecutionPayloadHeader::Bellatrix) } ForkName::Capella => { ExecutionPayloadHeaderCapella::::from_ssz_bytes(bytes.as_slice()) @@ -246,10 +246,10 @@ fn initialize_state_with_validators( ) -> Result, String> { // If no header is provided, then start from a Bellatrix state by default let default_header: ExecutionPayloadHeader = - ExecutionPayloadHeader::Merge(ExecutionPayloadHeaderMerge { + ExecutionPayloadHeader::Bellatrix(ExecutionPayloadHeaderBellatrix { block_hash: ExecutionBlockHash::from_root(eth1_block_hash), parent_hash: ExecutionBlockHash::zero(), - ..ExecutionPayloadHeaderMerge::default() + ..ExecutionPayloadHeaderBellatrix::default() }); let execution_payload_header = execution_payload_header.unwrap_or(default_header); // Empty eth1 data @@ -314,9 +314,9 @@ fn initialize_state_with_validators( // Override latest execution payload header. // See https://github.com/ethereum/consensus-specs/blob/v1.1.0/specs/bellatrix/beacon-chain.md#testing - if let ExecutionPayloadHeader::Merge(ref header) = execution_payload_header { + if let ExecutionPayloadHeader::Bellatrix(ref header) = execution_payload_header { *state - .latest_execution_payload_header_merge_mut() + .latest_execution_payload_header_bellatrix_mut() .or(Err("mismatched fork".to_string()))? = header.clone(); } } diff --git a/lcli/src/parse_ssz.rs b/lcli/src/parse_ssz.rs index 5b34fedfb9e..e86ffb73dc2 100644 --- a/lcli/src/parse_ssz.rs +++ b/lcli/src/parse_ssz.rs @@ -70,9 +70,11 @@ pub fn run_parse_ssz( "SignedBeaconBlockAltair" => { decode_and_print(&bytes, SignedBeaconBlockAltair::::from_ssz_bytes, format)? } - "SignedBeaconBlockMerge" | "SignedBeaconBlockBellatrix" => { - decode_and_print(&bytes, SignedBeaconBlockMerge::::from_ssz_bytes, format)? - } + "SignedBeaconBlockBellatrix" => decode_and_print( + &bytes, + SignedBeaconBlockBellatrix::::from_ssz_bytes, + format, + )?, "SignedBeaconBlockCapella" => decode_and_print( &bytes, SignedBeaconBlockCapella::::from_ssz_bytes, @@ -97,8 +99,8 @@ pub fn run_parse_ssz( "BeaconStateAltair" => { decode_and_print(&bytes, BeaconStateAltair::::from_ssz_bytes, format)? } - "BeaconStateMerge" | "BeaconStateBellatrix" => { - decode_and_print(&bytes, BeaconStateMerge::::from_ssz_bytes, format)? + "BeaconStateBellatrix" => { + decode_and_print(&bytes, BeaconStateBellatrix::::from_ssz_bytes, format)? } "BeaconStateCapella" => { decode_and_print(&bytes, BeaconStateCapella::::from_ssz_bytes, format)? diff --git a/lighthouse/environment/tests/testnet_dir/config.yaml b/lighthouse/environment/tests/testnet_dir/config.yaml index 86f4dce239b..c71feaa7dc5 100644 --- a/lighthouse/environment/tests/testnet_dir/config.yaml +++ b/lighthouse/environment/tests/testnet_dir/config.yaml @@ -32,7 +32,7 @@ GENESIS_DELAY: 604800 # Altair ALTAIR_FORK_VERSION: 0x01000000 ALTAIR_FORK_EPOCH: 18446744073709551615 -# Merge +# Bellatrix BELLATRIX_FORK_VERSION: 0x02000000 BELLATRIX_FORK_EPOCH: 18446744073709551615 # Sharding diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 9f1e3d235d9..7dfde69d3a9 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -416,7 +416,7 @@ fn eth1_cache_follow_distance_manual() { } // Tests for Bellatrix flags. -fn run_merge_execution_endpoints_flag_test(flag: &str) { +fn run_bellatrix_execution_endpoints_flag_test(flag: &str) { use sensitive_url::SensitiveUrl; let urls = vec!["http://sigp.io/no-way:1337", "http://infura.not_real:4242"]; // we don't support redundancy for execution-endpoints @@ -500,15 +500,15 @@ fn execution_timeout_multiplier_flag() { }); } #[test] -fn merge_execution_endpoints_flag() { - run_merge_execution_endpoints_flag_test("execution-endpoints") +fn bellatrix_execution_endpoints_flag() { + run_bellatrix_execution_endpoints_flag_test("execution-endpoints") } #[test] -fn merge_execution_endpoint_flag() { - run_merge_execution_endpoints_flag_test("execution-endpoint") +fn bellatrix_execution_endpoint_flag() { + run_bellatrix_execution_endpoints_flag_test("execution-endpoint") } #[test] -fn merge_jwt_secrets_flag() { +fn bellatrix_jwt_secrets_flag() { let dir = TempDir::new().expect("Unable to create temporary directory"); let mut file = File::create(dir.path().join("jwtsecrets")).expect("Unable to create file"); file.write_all(b"0x3cbc11b0d8fa16f3344eacfd6ff6430b9d30734450e8adcf5400f88d327dcb33") @@ -533,7 +533,7 @@ fn merge_jwt_secrets_flag() { }); } #[test] -fn merge_fee_recipient_flag() { +fn bellatrix_fee_recipient_flag() { let dir = TempDir::new().expect("Unable to create temporary directory"); CommandLineTest::new() .flag("execution-endpoint", Some("http://meow.cats")) diff --git a/testing/ef_tests/src/cases/common.rs b/testing/ef_tests/src/cases/common.rs index 342a48ba46c..6763edbe22b 100644 --- a/testing/ef_tests/src/cases/common.rs +++ b/testing/ef_tests/src/cases/common.rs @@ -63,8 +63,8 @@ pub fn previous_fork(fork_name: ForkName) -> ForkName { match fork_name { ForkName::Base => ForkName::Base, ForkName::Altair => ForkName::Base, - ForkName::Merge => ForkName::Altair, - ForkName::Capella => ForkName::Merge, + ForkName::Bellatrix => ForkName::Altair, + ForkName::Capella => ForkName::Bellatrix, ForkName::Deneb => ForkName::Capella, ForkName::Electra => ForkName::Deneb, } diff --git a/testing/ef_tests/src/cases/epoch_processing.rs b/testing/ef_tests/src/cases/epoch_processing.rs index a9c77c53c52..c4c592e4cf2 100644 --- a/testing/ef_tests/src/cases/epoch_processing.rs +++ b/testing/ef_tests/src/cases/epoch_processing.rs @@ -105,7 +105,7 @@ impl EpochTransition for JustificationAndFinalization { Ok(()) } BeaconState::Altair(_) - | BeaconState::Merge(_) + | BeaconState::Bellatrix(_) | BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) => { @@ -128,7 +128,7 @@ impl EpochTransition for RewardsAndPenalties { base::process_rewards_and_penalties(state, &validator_statuses, spec) } BeaconState::Altair(_) - | BeaconState::Merge(_) + | BeaconState::Bellatrix(_) | BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) => altair::process_rewards_and_penalties_slow(state, spec), @@ -161,7 +161,7 @@ impl EpochTransition for Slashings { )?; } BeaconState::Altair(_) - | BeaconState::Merge(_) + | BeaconState::Bellatrix(_) | BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) => { @@ -203,7 +203,7 @@ impl EpochTransition for RandaoMixesReset { impl EpochTransition for HistoricalRootsUpdate { fn run(state: &mut BeaconState, _spec: &ChainSpec) -> Result<(), EpochProcessingError> { match state { - BeaconState::Base(_) | BeaconState::Altair(_) | BeaconState::Merge(_) => { + BeaconState::Base(_) | BeaconState::Altair(_) | BeaconState::Bellatrix(_) => { process_historical_roots_update(state) } _ => Ok(()), @@ -237,7 +237,7 @@ impl EpochTransition for SyncCommitteeUpdates { match state { BeaconState::Base(_) => Ok(()), BeaconState::Altair(_) - | BeaconState::Merge(_) + | BeaconState::Bellatrix(_) | BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) => altair::process_sync_committee_updates(state, spec), @@ -250,7 +250,7 @@ impl EpochTransition for InactivityUpdates { match state { BeaconState::Base(_) => Ok(()), BeaconState::Altair(_) - | BeaconState::Merge(_) + | BeaconState::Bellatrix(_) | BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) => altair::process_inactivity_updates_slow(state, spec), @@ -263,7 +263,7 @@ impl EpochTransition for ParticipationFlagUpdates { match state { BeaconState::Base(_) => Ok(()), BeaconState::Altair(_) - | BeaconState::Merge(_) + | BeaconState::Bellatrix(_) | BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) => altair::process_participation_flag_updates(state), @@ -313,7 +313,7 @@ impl> Case for EpochProcessing { && T::name() != "historical_summaries_update" } // No phase0 tests for Altair and later. - ForkName::Altair | ForkName::Merge => { + ForkName::Altair | ForkName::Bellatrix => { T::name() != "participation_record_updates" && T::name() != "historical_summaries_update" } diff --git a/testing/ef_tests/src/cases/fork.rs b/testing/ef_tests/src/cases/fork.rs index be8a344a35a..132cfb4c0ae 100644 --- a/testing/ef_tests/src/cases/fork.rs +++ b/testing/ef_tests/src/cases/fork.rs @@ -63,7 +63,9 @@ impl Case for ForkTest { let mut result = match fork_name { ForkName::Base => panic!("phase0 not supported"), ForkName::Altair => upgrade_to_altair(&mut result_state, spec).map(|_| result_state), - ForkName::Merge => upgrade_to_bellatrix(&mut result_state, spec).map(|_| result_state), + ForkName::Bellatrix => { + upgrade_to_bellatrix(&mut result_state, spec).map(|_| result_state) + } ForkName::Capella => upgrade_to_capella(&mut result_state, spec).map(|_| result_state), ForkName::Deneb => upgrade_to_deneb(&mut result_state, spec).map(|_| result_state), ForkName::Electra => upgrade_to_electra(&mut result_state, spec).map(|_| result_state), diff --git a/testing/ef_tests/src/cases/merkle_proof_validity.rs b/testing/ef_tests/src/cases/merkle_proof_validity.rs index cf0b9f77c8f..8d5c0687753 100644 --- a/testing/ef_tests/src/cases/merkle_proof_validity.rs +++ b/testing/ef_tests/src/cases/merkle_proof_validity.rs @@ -92,7 +92,7 @@ pub struct KzgInclusionMerkleProofValidity { impl LoadCase for KzgInclusionMerkleProofValidity { fn load_from_dir(path: &Path, fork_name: ForkName) -> Result { let block: BeaconBlockBody> = match fork_name { - ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => { return Err(Error::InternalError(format!( "KZG inclusion merkle proof validity test skipped for {:?}", fork_name diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index a2f50896a57..158f2334dc3 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -20,8 +20,8 @@ use state_processing::{ }; use std::fmt::Debug; use types::{ - Attestation, AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockBodyCapella, - BeaconBlockBodyDeneb, BeaconBlockBodyMerge, BeaconState, BlindedPayload, Deposit, + Attestation, AttesterSlashing, BeaconBlock, BeaconBlockBody, BeaconBlockBodyBellatrix, + BeaconBlockBodyCapella, BeaconBlockBodyDeneb, BeaconState, BlindedPayload, Deposit, ExecutionPayload, FullPayload, ProposerSlashing, SignedBlsToExecutionChange, SignedVoluntaryExit, SyncAggregate, }; @@ -99,7 +99,7 @@ impl Operation for Attestation { spec, ), BeaconState::Altair(_) - | BeaconState::Merge(_) + | BeaconState::Bellatrix(_) | BeaconState::Capella(_) | BeaconState::Deneb(_) | BeaconState::Electra(_) => { @@ -289,7 +289,7 @@ impl Operation for BeaconBlockBody> { fn decode(path: &Path, fork_name: ForkName, _spec: &ChainSpec) -> Result { ssz_decode_file_with(path, |bytes| { Ok(match fork_name { - ForkName::Merge => BeaconBlockBody::Merge(<_>::from_ssz_bytes(bytes)?), + ForkName::Bellatrix => BeaconBlockBody::Bellatrix(<_>::from_ssz_bytes(bytes)?), ForkName::Capella => BeaconBlockBody::Capella(<_>::from_ssz_bytes(bytes)?), ForkName::Deneb => BeaconBlockBody::Deneb(<_>::from_ssz_bytes(bytes)?), _ => panic!(), @@ -330,9 +330,10 @@ impl Operation for BeaconBlockBody> { fn decode(path: &Path, fork_name: ForkName, _spec: &ChainSpec) -> Result { ssz_decode_file_with(path, |bytes| { Ok(match fork_name { - ForkName::Merge => { - let inner = >>::from_ssz_bytes(bytes)?; - BeaconBlockBody::Merge(inner.clone_as_blinded()) + ForkName::Bellatrix => { + let inner = + >>::from_ssz_bytes(bytes)?; + BeaconBlockBody::Bellatrix(inner.clone_as_blinded()) } ForkName::Capella => { let inner = >>::from_ssz_bytes(bytes)?; @@ -375,7 +376,9 @@ impl Operation for WithdrawalsPayload { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name != ForkName::Base && fork_name != ForkName::Altair && fork_name != ForkName::Merge + fork_name != ForkName::Base + && fork_name != ForkName::Altair + && fork_name != ForkName::Bellatrix } fn decode(path: &Path, fork_name: ForkName, _spec: &ChainSpec) -> Result { @@ -407,7 +410,9 @@ impl Operation for SignedBlsToExecutionChange { } fn is_enabled_for_fork(fork_name: ForkName) -> bool { - fork_name != ForkName::Base && fork_name != ForkName::Altair && fork_name != ForkName::Merge + fork_name != ForkName::Base + && fork_name != ForkName::Altair + && fork_name != ForkName::Bellatrix } fn decode(path: &Path, _fork_name: ForkName, _spec: &ChainSpec) -> Result { diff --git a/testing/ef_tests/src/cases/transition.rs b/testing/ef_tests/src/cases/transition.rs index b2c49a96feb..dc5029d53e7 100644 --- a/testing/ef_tests/src/cases/transition.rs +++ b/testing/ef_tests/src/cases/transition.rs @@ -38,7 +38,7 @@ impl LoadCase for TransitionTest { ForkName::Altair => { spec.altair_fork_epoch = Some(metadata.fork_epoch); } - ForkName::Merge => { + ForkName::Bellatrix => { spec.altair_fork_epoch = Some(Epoch::new(0)); spec.bellatrix_fork_epoch = Some(metadata.fork_epoch); } diff --git a/testing/ef_tests/src/handler.rs b/testing/ef_tests/src/handler.rs index 59b5cb6ba74..2d5ea4149ef 100644 --- a/testing/ef_tests/src/handler.rs +++ b/testing/ef_tests/src/handler.rs @@ -217,8 +217,8 @@ impl SszStaticHandler { Self::for_forks(vec![ForkName::Altair]) } - pub fn merge_only() -> Self { - Self::for_forks(vec![ForkName::Merge]) + pub fn bellatrix_only() -> Self { + Self::for_forks(vec![ForkName::Bellatrix]) } pub fn capella_only() -> Self { @@ -558,7 +558,7 @@ impl Handler for ForkChoiceHandler { fn is_enabled_for_fork(&self, fork_name: ForkName) -> bool { // Merge block tests are only enabled for Bellatrix. - if self.handler_name == "on_merge_block" && fork_name != ForkName::Merge { + if self.handler_name == "on_merge_block" && fork_name != ForkName::Bellatrix { return false; } @@ -823,7 +823,7 @@ impl Handler for KzgInclusionMerkleProofValidityHandler, MainnetEthSpec>::altair_only() .run(); - SszStaticHandler::, MinimalEthSpec>::merge_only() + SszStaticHandler::, MinimalEthSpec>::bellatrix_only() .run(); - SszStaticHandler::, MainnetEthSpec>::merge_only() + SszStaticHandler::, MainnetEthSpec>::bellatrix_only() .run(); SszStaticHandler::, MinimalEthSpec>::capella_only() .run(); @@ -290,10 +290,10 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::altair_only() .run(); - SszStaticHandler::, MinimalEthSpec>::merge_only( + SszStaticHandler::, MinimalEthSpec>::bellatrix_only( ) .run(); - SszStaticHandler::, MainnetEthSpec>::merge_only( + SszStaticHandler::, MainnetEthSpec>::bellatrix_only( ) .run(); SszStaticHandler::, MinimalEthSpec>::capella_only() @@ -313,9 +313,9 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::altair_only() .run(); - SszStaticHandler::, MinimalEthSpec>::merge_only() + SszStaticHandler::, MinimalEthSpec>::bellatrix_only() .run(); - SszStaticHandler::, MainnetEthSpec>::merge_only() + SszStaticHandler::, MainnetEthSpec>::bellatrix_only() .run(); SszStaticHandler::, MinimalEthSpec>::capella_only( @@ -340,10 +340,10 @@ mod ssz_static { SszStaticHandler::, MainnetEthSpec>::altair_only( ) .run(); - SszStaticHandler::, MinimalEthSpec>::merge_only( + SszStaticHandler::, MinimalEthSpec>::bellatrix_only( ) .run(); - SszStaticHandler::, MainnetEthSpec>::merge_only( + SszStaticHandler::, MainnetEthSpec>::bellatrix_only( ) .run(); SszStaticHandler::, MinimalEthSpec>::capella_only( @@ -369,10 +369,10 @@ mod ssz_static { SszStaticHandler::, MainnetEthSpec>::altair_only( ) .run(); - SszStaticHandler::, MinimalEthSpec>::merge_only( + SszStaticHandler::, MinimalEthSpec>::bellatrix_only( ) .run(); - SszStaticHandler::, MainnetEthSpec>::merge_only( + SszStaticHandler::, MainnetEthSpec>::bellatrix_only( ) .run(); SszStaticHandler::, MinimalEthSpec>::capella_only( @@ -396,9 +396,9 @@ mod ssz_static { .run(); SszStaticHandler::, MainnetEthSpec>::altair_only() .run(); - SszStaticHandler::, MinimalEthSpec>::merge_only() + SszStaticHandler::, MinimalEthSpec>::bellatrix_only() .run(); - SszStaticHandler::, MainnetEthSpec>::merge_only() + SszStaticHandler::, MainnetEthSpec>::bellatrix_only() .run(); SszStaticHandler::, MinimalEthSpec>::capella_only( ) @@ -448,12 +448,12 @@ mod ssz_static { SszStaticHandler::::altair_and_later().run(); } - // Merge and later + // Bellatrix and later #[test] fn execution_payload() { - SszStaticHandler::, MinimalEthSpec>::merge_only() + SszStaticHandler::, MinimalEthSpec>::bellatrix_only() .run(); - SszStaticHandler::, MainnetEthSpec>::merge_only() + SszStaticHandler::, MainnetEthSpec>::bellatrix_only() .run(); SszStaticHandler::, MinimalEthSpec>::capella_only() .run(); @@ -467,9 +467,9 @@ mod ssz_static { #[test] fn execution_payload_header() { - SszStaticHandler::, MinimalEthSpec>::merge_only() + SszStaticHandler::, MinimalEthSpec>::bellatrix_only() .run(); - SszStaticHandler::, MainnetEthSpec>::merge_only() + SszStaticHandler::, MainnetEthSpec>::bellatrix_only() .run(); SszStaticHandler::, MinimalEthSpec> ::capella_only().run(); diff --git a/testing/web3signer_tests/src/lib.rs b/testing/web3signer_tests/src/lib.rs index 3090b4da556..292e10d0542 100644 --- a/testing/web3signer_tests/src/lib.rs +++ b/testing/web3signer_tests/src/lib.rs @@ -724,11 +724,11 @@ mod tests { .await; } - /// Test all the Merge types. - async fn test_merge_types(network: &str, listen_port: u16) { + /// Test all the Bellatrix types. + async fn test_bellatrix_types(network: &str, listen_port: u16) { let network_config = Eth2NetworkConfig::constant(network).unwrap().unwrap(); let spec = &network_config.chain_spec::().unwrap(); - let merge_fork_slot = spec + let bellatrix_fork_slot = spec .bellatrix_fork_epoch .unwrap() .start_slot(E::slots_per_epoch()); @@ -740,14 +740,21 @@ mod tests { listen_port, ) .await - .assert_signatures_match("beacon_block_merge", |pubkey, validator_store| async move { - let mut merge_block = BeaconBlockMerge::empty(spec); - merge_block.slot = merge_fork_slot; - validator_store - .sign_block(pubkey, BeaconBlock::Merge(merge_block), merge_fork_slot) - .await - .unwrap() - }) + .assert_signatures_match( + "beacon_block_bellatrix", + |pubkey, validator_store| async move { + let mut bellatrix_block = BeaconBlockBellatrix::empty(spec); + bellatrix_block.slot = bellatrix_fork_slot; + validator_store + .sign_block( + pubkey, + BeaconBlock::Bellatrix(bellatrix_block), + bellatrix_fork_slot, + ) + .await + .unwrap() + }, + ) .await; } @@ -760,7 +767,7 @@ mod tests { let network_config = Eth2NetworkConfig::constant(network).unwrap().unwrap(); let spec = &network_config.chain_spec::().unwrap(); - let merge_fork_slot = spec + let bellatrix_fork_slot = spec .bellatrix_fork_epoch .unwrap() .start_slot(E::slots_per_epoch()); @@ -797,9 +804,9 @@ mod tests { }; let first_block = || { - let mut merge_block = BeaconBlockMerge::empty(spec); - merge_block.slot = merge_fork_slot; - BeaconBlock::Merge(merge_block) + let mut bellatrix_block = BeaconBlockBellatrix::empty(spec); + bellatrix_block.slot = bellatrix_fork_slot; + BeaconBlock::Bellatrix(bellatrix_block) }; let double_vote_block = || { @@ -914,8 +921,8 @@ mod tests { } #[tokio::test] - async fn sepolia_merge_types() { - test_merge_types("sepolia", 4252).await + async fn sepolia_bellatrix_types() { + test_bellatrix_types("sepolia", 4252).await } #[tokio::test] diff --git a/validator_client/src/signing_method/web3signer.rs b/validator_client/src/signing_method/web3signer.rs index 7973cab2c1e..8ad37a1620a 100644 --- a/validator_client/src/signing_method/web3signer.rs +++ b/validator_client/src/signing_method/web3signer.rs @@ -87,7 +87,7 @@ impl<'a, E: EthSpec, Payload: AbstractExecPayload> Web3SignerObject<'a, E, Pa block: Some(block), block_header: None, }), - BeaconBlock::Merge(_) => Ok(Web3SignerObject::BeaconBlock { + BeaconBlock::Bellatrix(_) => Ok(Web3SignerObject::BeaconBlock { version: ForkName::Bellatrix, block: None, block_header: Some(block.block_header()), diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index 0a00dad9beb..f3bdc2c0f69 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -360,7 +360,7 @@ impl ValidatorStore { fn signing_context(&self, domain: Domain, signing_epoch: Epoch) -> SigningContext { if domain == Domain::VoluntaryExit { match self.spec.fork_name_at_epoch(signing_epoch) { - ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + ForkName::Base | ForkName::Altair | ForkName::Bellatrix | ForkName::Capella => { SigningContext { domain, epoch: signing_epoch, diff --git a/watch/src/database/mod.rs b/watch/src/database/mod.rs index 59547c303ab..315fcbc8358 100644 --- a/watch/src/database/mod.rs +++ b/watch/src/database/mod.rs @@ -146,7 +146,7 @@ pub fn insert_beacon_block( let full_payload = block_message.execution_payload().ok(); let transaction_count: Option = if let Some(bellatrix_payload) = - full_payload.and_then(|payload| payload.execution_payload_merge().ok()) + full_payload.and_then(|payload| payload.execution_payload_bellatrix().ok()) { Some(bellatrix_payload.transactions.len() as i32) } else { From a1141ea1ef88dab78a85edd6440586c54b69475f Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Fri, 26 Apr 2024 22:25:45 +0900 Subject: [PATCH 32/37] Deterministic block generation for tests (#5654) * Deterministic block generation for tests --- .../types/src/test_utils/test_random/secret_key.rs | 2 ++ .../types/src/test_utils/test_random/signature.rs | 11 +++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/consensus/types/src/test_utils/test_random/secret_key.rs b/consensus/types/src/test_utils/test_random/secret_key.rs index 3f3f6ed5184..da1614aa24e 100644 --- a/consensus/types/src/test_utils/test_random/secret_key.rs +++ b/consensus/types/src/test_utils/test_random/secret_key.rs @@ -2,6 +2,8 @@ use super::*; impl TestRandom for SecretKey { fn random_for_test(_rng: &mut impl RngCore) -> Self { + // TODO: Not deterministic generation. Using `SecretKey::deserialize` results in + // `BlstError(BLST_BAD_ENCODING)`, need to debug with blst source on what encoding expects. SecretKey::random() } } diff --git a/consensus/types/src/test_utils/test_random/signature.rs b/consensus/types/src/test_utils/test_random/signature.rs index 5b952296b61..8bc0d711103 100644 --- a/consensus/types/src/test_utils/test_random/signature.rs +++ b/consensus/types/src/test_utils/test_random/signature.rs @@ -1,11 +1,10 @@ use super::*; impl TestRandom for Signature { - fn random_for_test(rng: &mut impl RngCore) -> Self { - let secret_key = SecretKey::random_for_test(rng); - let mut message = vec![0; 32]; - rng.fill_bytes(&mut message); - - secret_key.sign(Hash256::from_slice(&message)) + fn random_for_test(_rng: &mut impl RngCore) -> Self { + // TODO: `SecretKey::random_for_test` does not return a deterministic signature. Since this + // signature will not pass verification we could just return the generator point or the + // generator point multiplied by a random scalar if we want disctint signatures. + Signature::infinity().expect("infinity signature is valid") } } From 000a4fdf4db62fdc6b1cc3fc4c30301633d7d0d5 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Fri, 26 Apr 2024 16:17:23 -0400 Subject: [PATCH 33/37] Electra other containers (#5652) * add new fields to execution payload and header * beacon state changes * partial beacon state * safe arith in upgrade to electra * initialize balances cache in interop genesis state * Revert "initialize balances cache in interop genesis state" This reverts commit c60b522865177fe7f53c6b3be4e2e12567b27c53. * always initialize balances cache if necessary in electra upgrade * build cache earlier * fix block test * per fork NUM_FIELDS_POW2 * Merge branch 'unstable' of https://github.com/sigp/lighthouse into electra-other-containers * fix lints * get fields based on post state, as is spec'd * fix type and move cache build --- beacon_node/execution_layer/src/engine_api.rs | 3 + .../src/engine_api/json_structures.rs | 3 + beacon_node/execution_layer/src/lib.rs | 5 + .../test_utils/execution_block_generator.rs | 2 + beacon_node/store/src/partial_beacon_state.rs | 45 +++++- .../state_processing/src/upgrade/electra.rs | 62 ++++++++- consensus/types/src/beacon_block.rs | 5 +- consensus/types/src/beacon_state.rs | 130 +++++++++++++++++- consensus/types/src/beacon_state/tests.rs | 29 +--- consensus/types/src/execution_payload.rs | 5 + .../types/src/execution_payload_header.rs | 8 ++ consensus/types/src/validator.rs | 16 +++ 12 files changed, 275 insertions(+), 38 deletions(-) diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index f3f059a4355..ce1e0fec5dd 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -632,6 +632,9 @@ impl ExecutionPayloadBodyV1 { withdrawals, blob_gas_used: header.blob_gas_used, excess_blob_gas: header.excess_blob_gas, + // TODO(electra) + deposit_receipts: <_>::default(), + withdrawal_requests: <_>::default(), })) } else { Err(format!( diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index 5c4d6ab1ac3..50d3519e129 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -319,6 +319,9 @@ impl From> for ExecutionPayloadElectra .into(), blob_gas_used: payload.blob_gas_used, excess_blob_gas: payload.excess_blob_gas, + // TODO(electra) + deposit_receipts: Default::default(), + withdrawal_requests: Default::default(), } } } diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 668a5ce84b2..d441596edda 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -2003,6 +2003,11 @@ impl ExecutionLayer { withdrawals, blob_gas_used: electra_block.blob_gas_used, excess_blob_gas: electra_block.excess_blob_gas, + // TODO(electra) + // deposit_receipts: electra_block.deposit_receipts, + // withdrawal_requests: electra_block.withdrawal_requests, + deposit_receipts: <_>::default(), + withdrawal_requests: <_>::default(), }) } }; diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 77e12bcef66..e80c6b23705 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -659,6 +659,8 @@ impl ExecutionBlockGenerator { withdrawals: pa.withdrawals.clone().into(), blob_gas_used: 0, excess_blob_gas: 0, + deposit_receipts: vec![].into(), + withdrawal_requests: vec![].into(), }), _ => unreachable!(), }, diff --git a/beacon_node/store/src/partial_beacon_state.rs b/beacon_node/store/src/partial_beacon_state.rs index c2a15c0266f..e56d0580ac2 100644 --- a/beacon_node/store/src/partial_beacon_state.rs +++ b/beacon_node/store/src/partial_beacon_state.rs @@ -118,6 +118,29 @@ where #[ssz(skip_serializing, skip_deserializing)] #[superstruct(only(Capella, Deneb, Electra))] pub historical_summaries: Option>, + + // Electra + #[superstruct(only(Electra))] + pub deposit_receipts_start_index: u64, + #[superstruct(only(Electra))] + pub deposit_balance_to_consume: u64, + #[superstruct(only(Electra))] + pub exit_balance_to_consume: u64, + #[superstruct(only(Electra))] + pub earliest_exit_epoch: Epoch, + #[superstruct(only(Electra))] + pub consolidation_balance_to_consume: u64, + #[superstruct(only(Electra))] + pub earliest_consolidation_epoch: Epoch, + + // TODO(electra) should these be optional? + #[superstruct(only(Electra))] + pub pending_balance_deposits: List, + #[superstruct(only(Electra))] + pub pending_partial_withdrawals: + List, + #[superstruct(only(Electra))] + pub pending_consolidations: List, } /// Implement the conversion function from BeaconState -> PartialBeaconState. @@ -261,7 +284,16 @@ impl PartialBeaconState { inactivity_scores, latest_execution_payload_header, next_withdrawal_index, - next_withdrawal_validator_index + next_withdrawal_validator_index, + deposit_receipts_start_index, + deposit_balance_to_consume, + exit_balance_to_consume, + earliest_exit_epoch, + consolidation_balance_to_consume, + earliest_consolidation_epoch, + pending_balance_deposits, + pending_partial_withdrawals, + pending_consolidations ], [historical_summaries] ), @@ -525,7 +557,16 @@ impl TryInto> for PartialBeaconState { inactivity_scores, latest_execution_payload_header, next_withdrawal_index, - next_withdrawal_validator_index + next_withdrawal_validator_index, + deposit_receipts_start_index, + deposit_balance_to_consume, + exit_balance_to_consume, + earliest_exit_epoch, + consolidation_balance_to_consume, + earliest_consolidation_epoch, + pending_balance_deposits, + pending_partial_withdrawals, + pending_consolidations ], [historical_summaries] ), diff --git a/consensus/state_processing/src/upgrade/electra.rs b/consensus/state_processing/src/upgrade/electra.rs index f64228f050b..1e60bf488db 100644 --- a/consensus/state_processing/src/upgrade/electra.rs +++ b/consensus/state_processing/src/upgrade/electra.rs @@ -1,3 +1,4 @@ +use safe_arith::SafeArith; use std::mem; use types::{ BeaconState, BeaconStateElectra, BeaconStateError as Error, ChainSpec, EpochCache, EthSpec, @@ -10,14 +11,28 @@ pub fn upgrade_to_electra( spec: &ChainSpec, ) -> Result<(), Error> { let epoch = pre_state.current_epoch(); - let pre = pre_state.as_deneb_mut()?; + let earliest_exit_epoch = pre_state + .validators() + .iter() + .filter(|v| v.exit_epoch != spec.far_future_epoch) + .map(|v| v.exit_epoch) + .max() + .unwrap_or(epoch) + .safe_add(1)?; + + // The total active balance cache must be built before the consolidation churn limit + // is calculated. + pre_state.build_total_active_balance_cache(spec)?; + let earliest_consolidation_epoch = spec.compute_activation_exit_epoch(epoch)?; + + let pre = pre_state.as_deneb_mut()?; // Where possible, use something like `mem::take` to move fields from behind the &mut // reference. For other fields that don't have a good default value, use `clone`. // // Fixed size vectors get cloned because replacing them would require the same size // allocation as cloning. - let post = BeaconState::Electra(BeaconStateElectra { + let mut post = BeaconState::Electra(BeaconStateElectra { // Versioning genesis_time: pre.genesis_time, genesis_validators_root: pre.genesis_validators_root, @@ -62,6 +77,16 @@ pub fn upgrade_to_electra( next_withdrawal_index: pre.next_withdrawal_index, next_withdrawal_validator_index: pre.next_withdrawal_validator_index, historical_summaries: pre.historical_summaries.clone(), + // Electra + deposit_receipts_start_index: spec.unset_deposit_receipts_start_index, + deposit_balance_to_consume: 0, + exit_balance_to_consume: 0, + earliest_exit_epoch, + consolidation_balance_to_consume: 0, + earliest_consolidation_epoch, + pending_balance_deposits: Default::default(), + pending_partial_withdrawals: Default::default(), + pending_consolidations: Default::default(), // Caches total_active_balance: pre.total_active_balance, progressive_balances_cache: mem::take(&mut pre.progressive_balances_cache), @@ -71,6 +96,39 @@ pub fn upgrade_to_electra( slashings_cache: mem::take(&mut pre.slashings_cache), epoch_cache: EpochCache::default(), }); + *post.exit_balance_to_consume_mut()? = post.get_activation_exit_churn_limit(spec)?; + *post.consolidation_balance_to_consume_mut()? = post.get_consolidation_churn_limit(spec)?; + + // Add validators that are not yet active to pending balance deposits + let validators = post.validators().clone(); + let mut pre_activation = validators + .iter() + .enumerate() + .filter(|(_, validator)| validator.activation_epoch == spec.far_future_epoch) + .collect::>(); + + // Sort the indices by activation_eligibility_epoch and then by index + pre_activation.sort_by(|(index_a, val_a), (index_b, val_b)| { + if val_a.activation_eligibility_epoch == val_b.activation_eligibility_epoch { + index_a.cmp(index_b) + } else { + val_a + .activation_eligibility_epoch + .cmp(&val_b.activation_eligibility_epoch) + } + }); + + // Process validators to queue entire balance and reset them + for (index, _) in pre_activation { + post.queue_entire_balance_and_reset_validator(index, spec)?; + } + + // Ensure early adopters of compounding credentials go through the activation churn + for (index, validator) in validators.iter().enumerate() { + if validator.has_compounding_withdrawal_credential(spec) { + post.queue_excess_active_balance(index, spec)?; + } + } *pre_state = post; diff --git a/consensus/types/src/beacon_block.rs b/consensus/types/src/beacon_block.rs index 5af53b3fa17..81491d65056 100644 --- a/consensus/types/src/beacon_block.rs +++ b/consensus/types/src/beacon_block.rs @@ -1074,9 +1074,8 @@ mod tests { .expect("good electra block can be decoded"), good_block ); - // TODO(electra): once the Electra block is changed from Deneb, update this to match - // the other forks. - assert!(BeaconBlock::from_ssz_bytes(&bad_block.as_ssz_bytes(), &spec).is_ok()); + BeaconBlock::from_ssz_bytes(&bad_block.as_ssz_bytes(), &spec) + .expect_err("bad electra block cannot be decoded"); } } } diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index f6bae53505b..ef5cd23646b 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -467,6 +467,40 @@ where #[test_random(default)] pub historical_summaries: List, + // Electra + #[superstruct(only(Electra), partial_getter(copy))] + #[metastruct(exclude_from(tree_lists))] + #[serde(with = "serde_utils::quoted_u64")] + pub deposit_receipts_start_index: u64, + #[superstruct(only(Electra), partial_getter(copy))] + #[metastruct(exclude_from(tree_lists))] + #[serde(with = "serde_utils::quoted_u64")] + pub deposit_balance_to_consume: u64, + #[superstruct(only(Electra), partial_getter(copy))] + #[metastruct(exclude_from(tree_lists))] + #[serde(with = "serde_utils::quoted_u64")] + pub exit_balance_to_consume: u64, + #[superstruct(only(Electra), partial_getter(copy))] + #[metastruct(exclude_from(tree_lists))] + pub earliest_exit_epoch: Epoch, + #[superstruct(only(Electra), partial_getter(copy))] + #[metastruct(exclude_from(tree_lists))] + #[serde(with = "serde_utils::quoted_u64")] + pub consolidation_balance_to_consume: u64, + #[superstruct(only(Electra), partial_getter(copy))] + #[metastruct(exclude_from(tree_lists))] + pub earliest_consolidation_epoch: Epoch, + #[test_random(default)] + #[superstruct(only(Electra))] + pub pending_balance_deposits: List, + #[test_random(default)] + #[superstruct(only(Electra))] + pub pending_partial_withdrawals: + List, + #[test_random(default)] + #[superstruct(only(Electra))] + pub pending_consolidations: List, + // Caching (not in the spec) #[serde(skip_serializing, skip_deserializing)] #[ssz(skip_serializing, skip_deserializing)] @@ -2031,6 +2065,83 @@ impl BeaconState { self.epoch_cache().get_base_reward(validator_index) } + // ******* Electra accessors ******* + + /// Return the churn limit for the current epoch. + pub fn get_balance_churn_limit(&self, spec: &ChainSpec) -> Result { + let total_active_balance = self.get_total_active_balance()?; + let churn = std::cmp::max( + spec.min_per_epoch_churn_limit_electra, + total_active_balance.safe_div(spec.churn_limit_quotient)?, + ); + + Ok(churn.safe_sub(churn.safe_rem(spec.effective_balance_increment)?)?) + } + + /// Return the churn limit for the current epoch dedicated to activations and exits. + pub fn get_activation_exit_churn_limit(&self, spec: &ChainSpec) -> Result { + Ok(std::cmp::min( + spec.max_per_epoch_activation_exit_churn_limit, + self.get_balance_churn_limit(spec)?, + )) + } + + pub fn get_consolidation_churn_limit(&self, spec: &ChainSpec) -> Result { + self.get_balance_churn_limit(spec)? + .safe_sub(self.get_activation_exit_churn_limit(spec)?) + .map_err(Into::into) + } + + // ******* Electra mutators ******* + + pub fn queue_excess_active_balance( + &mut self, + validator_index: usize, + spec: &ChainSpec, + ) -> Result<(), Error> { + let balance = self + .balances_mut() + .get_mut(validator_index) + .ok_or(Error::UnknownValidator(validator_index))?; + if *balance > spec.min_activation_balance { + let excess_balance = balance.safe_sub(spec.min_activation_balance)?; + *balance = spec.min_activation_balance; + self.pending_balance_deposits_mut()? + .push(PendingBalanceDeposit { + index: validator_index as u64, + amount: excess_balance, + })?; + } + Ok(()) + } + + pub fn queue_entire_balance_and_reset_validator( + &mut self, + validator_index: usize, + spec: &ChainSpec, + ) -> Result<(), Error> { + let balance = self + .balances_mut() + .get_mut(validator_index) + .ok_or(Error::UnknownValidator(validator_index))?; + let balance_copy = *balance; + *balance = 0_u64; + + let validator = self + .validators_mut() + .get_mut(validator_index) + .ok_or(Error::UnknownValidator(validator_index))?; + validator.effective_balance = 0; + validator.activation_eligibility_epoch = spec.far_future_epoch; + + self.pending_balance_deposits_mut()? + .push(PendingBalanceDeposit { + index: validator_index as u64, + amount: balance_copy, + }) + .map_err(Into::into) + } + #[allow(clippy::arithmetic_side_effects)] pub fn rebase_on(&mut self, base: &Self, spec: &ChainSpec) -> Result<(), Error> { // Required for macros (which use type-hints internally). @@ -2147,10 +2258,17 @@ impl BeaconState { /// The number of fields of the `BeaconState` rounded up to the nearest power of two. /// /// This is relevant to tree-hashing of the `BeaconState`. - /// - /// We assume this value is stable across forks. This assumption is checked in the - /// `check_num_fields_pow2` test. - pub const NUM_FIELDS_POW2: usize = BeaconStateBellatrix::::NUM_FIELDS.next_power_of_two(); + pub fn num_fields_pow2(&self) -> usize { + let fork_name = self.fork_name_unchecked(); + match fork_name { + ForkName::Base => BeaconStateBase::::NUM_FIELDS.next_power_of_two(), + ForkName::Altair => BeaconStateAltair::::NUM_FIELDS.next_power_of_two(), + ForkName::Bellatrix => BeaconStateBellatrix::::NUM_FIELDS.next_power_of_two(), + ForkName::Capella => BeaconStateCapella::::NUM_FIELDS.next_power_of_two(), + ForkName::Deneb => BeaconStateDeneb::::NUM_FIELDS.next_power_of_two(), + ForkName::Electra => BeaconStateElectra::::NUM_FIELDS.next_power_of_two(), + } + } /// Specialised deserialisation method that uses the `ChainSpec` as context. #[allow(clippy::arithmetic_side_effects)] @@ -2211,7 +2329,7 @@ impl BeaconState { // in the `BeaconState`: // https://github.com/ethereum/consensus-specs/blob/dev/specs/altair/beacon-chain.md#beaconstate generalized_index - .checked_sub(Self::NUM_FIELDS_POW2) + .checked_sub(self.num_fields_pow2()) .ok_or(Error::IndexNotSupported(generalized_index))? } light_client_update::FINALIZED_ROOT_INDEX => { @@ -2221,7 +2339,7 @@ impl BeaconState { // Subtract off the internal nodes. Result should be 105/2 - 32 = 20 which matches // position of `finalized_checkpoint` in `BeaconState`. finalized_checkpoint_generalized_index - .checked_sub(Self::NUM_FIELDS_POW2) + .checked_sub(self.num_fields_pow2()) .ok_or(Error::IndexNotSupported(generalized_index))? } _ => return Err(Error::IndexNotSupported(generalized_index)), diff --git a/consensus/types/src/beacon_state/tests.rs b/consensus/types/src/beacon_state/tests.rs index 35afe314957..38a76e44c50 100644 --- a/consensus/types/src/beacon_state/tests.rs +++ b/consensus/types/src/beacon_state/tests.rs @@ -1,10 +1,10 @@ #![cfg(test)] -use crate::{test_utils::*, ForkName}; +use crate::test_utils::*; use beacon_chain::test_utils::{BeaconChainHarness, EphemeralHarnessType}; use beacon_chain::types::{ - test_utils::TestRandom, BeaconState, BeaconStateAltair, BeaconStateBase, BeaconStateBellatrix, - BeaconStateCapella, BeaconStateDeneb, BeaconStateElectra, BeaconStateError, ChainSpec, Domain, - Epoch, EthSpec, Hash256, Keypair, MainnetEthSpec, MinimalEthSpec, RelativeEpoch, Slot, Vector, + test_utils::TestRandom, BeaconState, BeaconStateAltair, BeaconStateBase, BeaconStateError, + ChainSpec, Domain, Epoch, EthSpec, Hash256, Keypair, MainnetEthSpec, MinimalEthSpec, + RelativeEpoch, Slot, Vector, }; use ssz::Encode; use std::ops::Mul; @@ -403,24 +403,3 @@ fn decode_base_and_altair() { .expect_err("bad altair state cannot be decoded"); } } - -#[test] -fn check_num_fields_pow2() { - use metastruct::NumFields; - pub type E = MainnetEthSpec; - - for fork_name in ForkName::list_all() { - let num_fields = match fork_name { - ForkName::Base => BeaconStateBase::::NUM_FIELDS, - ForkName::Altair => BeaconStateAltair::::NUM_FIELDS, - ForkName::Bellatrix => BeaconStateBellatrix::::NUM_FIELDS, - ForkName::Capella => BeaconStateCapella::::NUM_FIELDS, - ForkName::Deneb => BeaconStateDeneb::::NUM_FIELDS, - ForkName::Electra => BeaconStateElectra::::NUM_FIELDS, - }; - assert_eq!( - num_fields.next_power_of_two(), - BeaconState::::NUM_FIELDS_POW2 - ); - } -} diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index a46e7c29fff..0946b9ecffa 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -89,6 +89,11 @@ pub struct ExecutionPayload { #[superstruct(only(Deneb, Electra), partial_getter(copy))] #[serde(with = "serde_utils::quoted_u64")] pub excess_blob_gas: u64, + #[superstruct(only(Electra))] + pub deposit_receipts: VariableList, + #[superstruct(only(Electra))] + pub withdrawal_requests: + VariableList, } impl<'a, E: EthSpec> ExecutionPayloadRef<'a, E> { diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index 8515e386c22..324d7b97472 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -88,6 +88,10 @@ pub struct ExecutionPayloadHeader { #[serde(with = "serde_utils::quoted_u64")] #[superstruct(getter(copy))] pub excess_blob_gas: u64, + #[superstruct(only(Electra), partial_getter(copy))] + pub deposit_receipts_root: Hash256, + #[superstruct(only(Electra), partial_getter(copy))] + pub withdrawal_requests_root: Hash256, } impl ExecutionPayloadHeader { @@ -206,6 +210,8 @@ impl ExecutionPayloadHeaderDeneb { withdrawals_root: self.withdrawals_root, blob_gas_used: self.blob_gas_used, excess_blob_gas: self.excess_blob_gas, + deposit_receipts_root: Hash256::zero(), + withdrawal_requests_root: Hash256::zero(), } } } @@ -297,6 +303,8 @@ impl<'a, E: EthSpec> From<&'a ExecutionPayloadElectra> for ExecutionPayloadHe withdrawals_root: payload.withdrawals.tree_hash_root(), blob_gas_used: payload.blob_gas_used, excess_blob_gas: payload.excess_blob_gas, + deposit_receipts_root: payload.deposit_receipts.tree_hash_root(), + withdrawal_requests_root: payload.withdrawal_requests.tree_hash_root(), } } } diff --git a/consensus/types/src/validator.rs b/consensus/types/src/validator.rs index 98567cd1e6c..5fd18552f6b 100644 --- a/consensus/types/src/validator.rs +++ b/consensus/types/src/validator.rs @@ -102,6 +102,11 @@ impl Validator { .unwrap_or(false) } + /// Check if ``validator`` has an 0x02 prefixed "compounding" withdrawal credential. + pub fn has_compounding_withdrawal_credential(&self, spec: &ChainSpec) -> bool { + is_compounding_withdrawal_credential(self.withdrawal_credentials, spec) + } + /// Get the eth1 withdrawal address if this validator has one initialized. pub fn get_eth1_withdrawal_address(&self, spec: &ChainSpec) -> Option
{ self.has_eth1_withdrawal_credential(spec) @@ -153,6 +158,17 @@ impl Default for Validator { } } +pub fn is_compounding_withdrawal_credential( + withdrawal_credentials: Hash256, + spec: &ChainSpec, +) -> bool { + withdrawal_credentials + .as_bytes() + .first() + .map(|prefix_byte| *prefix_byte == spec.compounding_withdrawal_prefix_byte) + .unwrap_or(false) +} + #[cfg(test)] mod tests { use super::*; From 8b24880df78659ab1a58f37859aeabbad838dae7 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Sat, 27 Apr 2024 06:39:29 +0530 Subject: [PATCH 34/37] Add more electra helpers (#5653) * Add new helpers * Fix some stuff * Fix compilation errors * lint * Address review --- consensus/types/src/beacon_state.rs | 127 ++++++++++++++++++++++++++++ consensus/types/src/validator.rs | 36 ++++++-- 2 files changed, 157 insertions(+), 6 deletions(-) diff --git a/consensus/types/src/beacon_state.rs b/consensus/types/src/beacon_state.rs index ef5cd23646b..577f282a556 100644 --- a/consensus/types/src/beacon_state.rs +++ b/consensus/types/src/beacon_state.rs @@ -2092,6 +2092,38 @@ impl BeaconState { .map_err(Into::into) } + /// Get active balance for the given `validator_index`. + pub fn get_active_balance( + &self, + validator_index: usize, + spec: &ChainSpec, + ) -> Result { + let max_effective_balance = self + .validators() + .get(validator_index) + .map(|validator| validator.get_validator_max_effective_balance(spec)) + .ok_or(Error::UnknownValidator(validator_index))?; + Ok(std::cmp::min( + *self + .balances() + .get(validator_index) + .ok_or(Error::UnknownValidator(validator_index))?, + max_effective_balance, + )) + } + + pub fn get_pending_balance_to_withdraw(&self, validator_index: usize) -> Result { + let mut pending_balance = 0; + for withdrawal in self + .pending_partial_withdrawals()? + .iter() + .filter(|withdrawal| withdrawal.index as usize == validator_index) + { + pending_balance.safe_add_assign(withdrawal.amount)?; + } + Ok(pending_balance) + } + // ******* Electra mutators ******* pub fn queue_excess_active_balance( @@ -2142,6 +2174,101 @@ impl BeaconState { .map_err(Into::into) } + /// Change the withdrawal prefix of the given `validator_index` to the compounding withdrawal validator prefix. + pub fn switch_to_compounding_validator( + &mut self, + validator_index: usize, + spec: &ChainSpec, + ) -> Result<(), Error> { + let validator = self + .validators_mut() + .get_mut(validator_index) + .ok_or(Error::UnknownValidator(validator_index))?; + if validator.has_eth1_withdrawal_credential(spec) { + validator.withdrawal_credentials.as_fixed_bytes_mut()[0] = + spec.compounding_withdrawal_prefix_byte; + self.queue_excess_active_balance(validator_index, spec)?; + } + Ok(()) + } + + pub fn compute_exit_epoch_and_update_churn( + &mut self, + exit_balance: u64, + spec: &ChainSpec, + ) -> Result { + let mut earliest_exit_epoch = std::cmp::max( + self.earliest_exit_epoch()?, + self.compute_activation_exit_epoch(self.current_epoch(), spec)?, + ); + + let per_epoch_churn = self.get_activation_exit_churn_limit(spec)?; + // New epoch for exits + let mut exit_balance_to_consume = if self.earliest_exit_epoch()? < earliest_exit_epoch { + per_epoch_churn + } else { + self.exit_balance_to_consume()? + }; + + // Exit doesn't fit in the current earliest epoch + if exit_balance > exit_balance_to_consume { + let balance_to_process = exit_balance.safe_sub(exit_balance_to_consume)?; + let additional_epochs = balance_to_process + .safe_sub(1)? + .safe_div(per_epoch_churn)? + .safe_add(1)?; + earliest_exit_epoch.safe_add_assign(additional_epochs)?; + exit_balance_to_consume + .safe_add_assign(additional_epochs.safe_mul(per_epoch_churn)?)?; + } + let state = self.as_electra_mut()?; + // Consume the balance and update state variables + state.exit_balance_to_consume = exit_balance_to_consume.safe_sub(exit_balance)?; + state.earliest_exit_epoch = earliest_exit_epoch; + + Ok(state.earliest_exit_epoch) + } + + pub fn compute_consolidation_epoch_and_update_churn( + &mut self, + consolidation_balance: u64, + spec: &ChainSpec, + ) -> Result { + let mut earliest_consolidation_epoch = std::cmp::max( + self.earliest_consolidation_epoch()?, + self.compute_activation_exit_epoch(self.current_epoch(), spec)?, + ); + + let per_epoch_consolidation_churn = self.get_consolidation_churn_limit(spec)?; + + // New epoch for consolidations + let mut consolidation_balance_to_consume = + if self.earliest_consolidation_epoch()? < earliest_consolidation_epoch { + per_epoch_consolidation_churn + } else { + self.consolidation_balance_to_consume()? + }; + // Consolidation doesn't fit in the current earliest epoch + if consolidation_balance > consolidation_balance_to_consume { + let balance_to_process = + consolidation_balance.safe_sub(consolidation_balance_to_consume)?; + let additional_epochs = balance_to_process + .safe_sub(1)? + .safe_div(per_epoch_consolidation_churn)? + .safe_add(1)?; + earliest_consolidation_epoch.safe_add_assign(additional_epochs)?; + consolidation_balance_to_consume + .safe_add_assign(additional_epochs.safe_mul(per_epoch_consolidation_churn)?)?; + } + // Consume the balance and update state variables + let state = self.as_electra_mut()?; + state.consolidation_balance_to_consume = + consolidation_balance_to_consume.safe_sub(consolidation_balance)?; + state.earliest_consolidation_epoch = earliest_consolidation_epoch; + + Ok(state.earliest_consolidation_epoch) + } + #[allow(clippy::arithmetic_side_effects)] pub fn rebase_on(&mut self, base: &Self, spec: &ChainSpec) -> Result<(), Error> { // Required for macros (which use type-hints internally). diff --git a/consensus/types/src/validator.rs b/consensus/types/src/validator.rs index 5fd18552f6b..8ed449ec8a7 100644 --- a/consensus/types/src/validator.rs +++ b/consensus/types/src/validator.rs @@ -57,10 +57,10 @@ impl Validator { /// Returns `true` if the validator is eligible to join the activation queue. /// - /// Spec v0.12.1 + /// Modified in electra pub fn is_eligible_for_activation_queue(&self, spec: &ChainSpec) -> bool { self.activation_eligibility_epoch == spec.far_future_epoch - && self.effective_balance == spec.max_effective_balance + && self.effective_balance >= spec.min_activation_balance } /// Returns `true` if the validator is eligible to be activated. @@ -130,15 +130,39 @@ impl Validator { } /// Returns `true` if the validator is fully withdrawable at some epoch. + /// + /// Note: Modified in electra. pub fn is_fully_withdrawable_at(&self, balance: u64, epoch: Epoch, spec: &ChainSpec) -> bool { - self.has_eth1_withdrawal_credential(spec) && self.withdrawable_epoch <= epoch && balance > 0 + self.has_execution_withdrawal_credential(spec) + && self.withdrawable_epoch <= epoch + && balance > 0 } /// Returns `true` if the validator is partially withdrawable. + /// + /// Note: Modified in electra. pub fn is_partially_withdrawable_validator(&self, balance: u64, spec: &ChainSpec) -> bool { - self.has_eth1_withdrawal_credential(spec) - && self.effective_balance == spec.max_effective_balance - && balance > spec.max_effective_balance + let max_effective_balance = self.get_validator_max_effective_balance(spec); + let has_max_effective_balance = self.effective_balance == max_effective_balance; + let has_excess_balance = balance > max_effective_balance; + self.has_execution_withdrawal_credential(spec) + && has_max_effective_balance + && has_excess_balance + } + + /// Returns `true` if the validator has a 0x01 or 0x02 prefixed withdrawal credential. + pub fn has_execution_withdrawal_credential(&self, spec: &ChainSpec) -> bool { + self.has_compounding_withdrawal_credential(spec) + || self.has_eth1_withdrawal_credential(spec) + } + + /// Returns the max effective balance for a validator in gwei. + pub fn get_validator_max_effective_balance(&self, spec: &ChainSpec) -> u64 { + if self.has_compounding_withdrawal_credential(spec) { + spec.max_effective_balance_electra + } else { + spec.min_activation_balance + } } } From 40d412629ecfad250d6f29dc540b50ab4e399639 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 29 Apr 2024 23:30:13 +0900 Subject: [PATCH 35/37] Ignore gossip blob already imported (#5656) * Ignore gossip blob already imported --- .../network_beacon_processor/gossip_methods.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 7b8826bd853..af7f3a53e56 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -778,24 +778,28 @@ impl NetworkBeaconProcessor { "block_root" => %block_root, ); } + Err(BlockError::BlockIsAlreadyKnown(_)) => { + debug!( + self.log, + "Ignoring gossip blob already imported"; + "block_root" => ?block_root, + "blob_index" => blob_index, + ); + } Err(err) => { debug!( self.log, "Invalid gossip blob"; "outcome" => ?err, - "block root" => ?block_root, - "block slot" => blob_slot, - "blob index" => blob_index, + "block_root" => ?block_root, + "block_slot" => blob_slot, + "blob_index" => blob_index, ); self.gossip_penalize_peer( peer_id, PeerAction::MidToleranceError, "bad_gossip_blob_ssz", ); - trace!( - self.log, - "Invalid gossip blob ssz"; - ); } } } From c33edc82ebcf0d6db45d8c715cbb071e0d62fcd9 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 29 Apr 2024 16:45:54 -0400 Subject: [PATCH 36/37] Beta compiler fix (#5659) * fix beta compiler compilation * remove unused import * Revert "remove unused import" This reverts commit 0bef36b05bb146cfcc4a2e0b77ddbd8852e3bf9f. * Revert "fix beta compiler compilation" This reverts commit 23152cf4cc1ee627b76585a013d093f14e45ab14. * rename ununsed fields * allow dead code on some error variants * remove unused blob download queue * add back debug to backfill error * more allow dead code on errors --- .../network/src/sync/backfill_sync/mod.rs | 9 +++++---- .../network/src/sync/block_lookups/common.rs | 2 +- .../sync/block_lookups/single_block_lookup.rs | 10 +++------- beacon_node/network/src/sync/manager.rs | 16 ++++++++-------- common/logging/src/lib.rs | 8 ++++---- common/logging/src/tracing_logging_layer.rs | 4 ++-- validator_client/src/duties_service.rs | 11 ++++++----- validator_client/src/http_metrics/mod.rs | 4 ++-- 8 files changed, 31 insertions(+), 33 deletions(-) diff --git a/beacon_node/network/src/sync/backfill_sync/mod.rs b/beacon_node/network/src/sync/backfill_sync/mod.rs index 67fe871accf..9075bb15f08 100644 --- a/beacon_node/network/src/sync/backfill_sync/mod.rs +++ b/beacon_node/network/src/sync/backfill_sync/mod.rs @@ -87,16 +87,17 @@ pub enum ProcessResult { } /// The ways a backfill sync can fail. +// The info in the enum variants is displayed in logging, clippy thinks it's dead code. #[derive(Debug)] pub enum BackFillError { /// A batch failed to be downloaded. - BatchDownloadFailed(BatchId), + BatchDownloadFailed(#[allow(dead_code)] BatchId), /// A batch could not be processed. - BatchProcessingFailed(BatchId), + BatchProcessingFailed(#[allow(dead_code)] BatchId), /// A batch entered an invalid state. - BatchInvalidState(BatchId, String), + BatchInvalidState(#[allow(dead_code)] BatchId, #[allow(dead_code)] String), /// The sync algorithm entered an invalid state. - InvalidSyncState(String), + InvalidSyncState(#[allow(dead_code)] String), /// The chain became paused. Paused, } diff --git a/beacon_node/network/src/sync/block_lookups/common.rs b/beacon_node/network/src/sync/block_lookups/common.rs index 7193dd6e216..9d1af0cb813 100644 --- a/beacon_node/network/src/sync/block_lookups/common.rs +++ b/beacon_node/network/src/sync/block_lookups/common.rs @@ -245,7 +245,7 @@ impl RequestState for BlockRequestState { } } -impl RequestState for BlobRequestState { +impl RequestState for BlobRequestState { type RequestType = BlobsByRootSingleBlockRequest; type VerifiedResponseType = FixedBlobSidecarList; type ReconstructedResponseType = FixedBlobSidecarList; diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index 077af7c3d19..49ef1dd15bf 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -18,7 +18,6 @@ use std::fmt::Debug; use std::sync::Arc; use store::Hash256; use strum::IntoStaticStr; -use types::blob_sidecar::FixedBlobSidecarList; use types::EthSpec; #[derive(Debug, PartialEq, Eq, IntoStaticStr)] @@ -37,7 +36,7 @@ pub struct SingleBlockLookup { pub id: Id, pub lookup_type: LookupType, pub block_request_state: BlockRequestState, - pub blob_request_state: BlobRequestState, + pub blob_request_state: BlobRequestState, pub da_checker: Arc>, /// Only necessary for requests triggered by an `UnknownBlockParent` or `UnknownBlockParent` /// because any blocks or blobs without parents won't hit the data availability cache. @@ -304,24 +303,21 @@ impl SingleBlockLookup { } /// The state of the blob request component of a `SingleBlockLookup`. -pub struct BlobRequestState { +pub struct BlobRequestState { /// The latest picture of which blobs still need to be requested. This includes information /// from both block/blobs downloaded in the network layer and any blocks/blobs that exist in /// the data availability checker. pub requested_ids: MissingBlobs, pub block_root: Hash256, - /// Where we store blobs until we receive the stream terminator. - pub blob_download_queue: FixedBlobSidecarList, pub state: SingleLookupRequestState, } -impl BlobRequestState { +impl BlobRequestState { pub fn new(block_root: Hash256, peer_source: &[PeerId], is_deneb: bool) -> Self { let default_ids = MissingBlobs::new_without_block(block_root, is_deneb); Self { block_root, requested_ids: default_ids, - blob_download_queue: <_>::default(), state: SingleLookupRequestState::new(peer_source), } } diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 9c17c6a1512..15b96c52b10 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -627,7 +627,7 @@ impl SyncManager { ), BlockProcessType::SingleBlob { id } => self .block_lookups - .single_block_component_processed::>( + .single_block_component_processed::( id, result, &mut self.network, @@ -908,7 +908,7 @@ impl SyncManager { Ok((blobs, seen_timestamp)) => match id.lookup_type { LookupType::Current => self .block_lookups - .single_lookup_response::>( + .single_lookup_response::( id, peer_id, blobs, @@ -917,7 +917,7 @@ impl SyncManager { ), LookupType::Parent => self .block_lookups - .parent_lookup_response::>( + .parent_lookup_response::( id, peer_id, blobs, @@ -929,20 +929,20 @@ impl SyncManager { Err(error) => match id.lookup_type { LookupType::Current => self .block_lookups - .single_block_lookup_failed::>( + .single_block_lookup_failed::( id, &peer_id, &mut self.network, error, ), - LookupType::Parent => self - .block_lookups - .parent_lookup_failed::>( + LookupType::Parent => { + self.block_lookups.parent_lookup_failed::( id, &peer_id, &mut self.network, error, - ), + ) + } }, } } diff --git a/common/logging/src/lib.rs b/common/logging/src/lib.rs index 3a5a5209b02..b0e1da00e97 100644 --- a/common/logging/src/lib.rs +++ b/common/logging/src/lib.rs @@ -256,14 +256,14 @@ pub fn create_tracing_layer(base_tracing_log_path: PathBuf) { return; }; - let (libp2p_non_blocking_writer, libp2p_guard) = NonBlocking::new(libp2p_writer); - let (discv5_non_blocking_writer, discv5_guard) = NonBlocking::new(discv5_writer); + let (libp2p_non_blocking_writer, _libp2p_guard) = NonBlocking::new(libp2p_writer); + let (discv5_non_blocking_writer, _discv5_guard) = NonBlocking::new(discv5_writer); let custom_layer = LoggingLayer { libp2p_non_blocking_writer, - libp2p_guard, + _libp2p_guard, discv5_non_blocking_writer, - discv5_guard, + _discv5_guard, }; if let Err(e) = tracing_subscriber::fmt() diff --git a/common/logging/src/tracing_logging_layer.rs b/common/logging/src/tracing_logging_layer.rs index aabb6ddd0c9..a9ddae828ae 100644 --- a/common/logging/src/tracing_logging_layer.rs +++ b/common/logging/src/tracing_logging_layer.rs @@ -7,9 +7,9 @@ use tracing_subscriber::Layer; pub struct LoggingLayer { pub libp2p_non_blocking_writer: NonBlocking, - pub libp2p_guard: WorkerGuard, + pub _libp2p_guard: WorkerGuard, pub discv5_non_blocking_writer: NonBlocking, - pub discv5_guard: WorkerGuard, + pub _discv5_guard: WorkerGuard, } impl Layer for LoggingLayer diff --git a/validator_client/src/duties_service.rs b/validator_client/src/duties_service.rs index b5b56943c4b..15e5c45e0d5 100644 --- a/validator_client/src/duties_service.rs +++ b/validator_client/src/duties_service.rs @@ -88,14 +88,15 @@ const _: () = assert!({ /// bringing in the entire crate. const _: () = assert!(ATTESTATION_SUBSCRIPTION_OFFSETS[0] > 2); +// The info in the enum variants is displayed in logging, clippy thinks it's dead code. #[derive(Debug)] pub enum Error { UnableToReadSlotClock, - FailedToDownloadAttesters(String), - FailedToProduceSelectionProof(ValidatorStoreError), - InvalidModulo(ArithError), - Arith(ArithError), - SyncDutiesNotFound(u64), + FailedToDownloadAttesters(#[allow(dead_code)] String), + FailedToProduceSelectionProof(#[allow(dead_code)] ValidatorStoreError), + InvalidModulo(#[allow(dead_code)] ArithError), + Arith(#[allow(dead_code)] ArithError), + SyncDutiesNotFound(#[allow(dead_code)] u64), } impl From for Error { diff --git a/validator_client/src/http_metrics/mod.rs b/validator_client/src/http_metrics/mod.rs index de6c06437b4..67cab2bdc37 100644 --- a/validator_client/src/http_metrics/mod.rs +++ b/validator_client/src/http_metrics/mod.rs @@ -17,8 +17,8 @@ use warp::{http::Response, Filter}; #[derive(Debug)] pub enum Error { - Warp(warp::Error), - Other(String), + Warp(#[allow(dead_code)] warp::Error), + Other(#[allow(dead_code)] String), } impl From for Error { From c8ffafb3f8e5022661020354d727b12b573b9e88 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Tue, 30 Apr 2024 00:55:23 +0200 Subject: [PATCH 37/37] fix(validator_client): raise soft fd limit (#4796) * fix(validator_client): raise soft fd limit * Merge branch 'unstable' of https://github.com/sigp/lighthouse into rkrasiuk/raise-vc-fdlimit * cargo lock * Merge branch 'unstable' of https://github.com/sigp/lighthouse into rkrasiuk/raise-vc-fdlimit --- Cargo.lock | 11 +++++++++++ validator_client/Cargo.toml | 1 + validator_client/src/lib.rs | 23 ++++++++++++++++++++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index cc38877fe76..85c89528d0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2912,6 +2912,16 @@ dependencies = [ "bytes", ] +[[package]] +name = "fdlimit" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e182f7dbc2ef73d9ef67351c5fbbea084729c48362d3ce9dd44c28e32e277fe5" +dependencies = [ + "libc", + "thiserror", +] + [[package]] name = "ff" version = "0.12.1" @@ -8988,6 +8998,7 @@ dependencies = [ "eth2", "eth2_keystore", "ethereum_serde_utils", + "fdlimit", "filesystem", "futures", "hex", diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index d3dffc3d02e..0df687abec5 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -61,3 +61,4 @@ sysinfo = { workspace = true } system_health = { path = "../common/system_health" } logging = { workspace = true } strum = { workspace = true } +fdlimit = "0.3.0" diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 268c25cdf7d..377b0640483 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -47,7 +47,7 @@ use notifier::spawn_notifier; use parking_lot::RwLock; use preparation_service::{PreparationService, PreparationServiceBuilder}; use reqwest::Certificate; -use slog::{error, info, warn, Logger}; +use slog::{debug, error, info, warn, Logger}; use slot_clock::SlotClock; use slot_clock::SystemTimeSlotClock; use std::fs::File; @@ -121,6 +121,27 @@ impl ProductionValidatorClient { pub async fn new(context: RuntimeContext, config: Config) -> Result { let log = context.log().clone(); + // Attempt to raise soft fd limit. The behavior is OS specific: + // `linux` - raise soft fd limit to hard + // `macos` - raise soft fd limit to `min(kernel limit, hard fd limit)` + // `windows` & rest - noop + match fdlimit::raise_fd_limit().map_err(|e| format!("Unable to raise fd limit: {}", e))? { + fdlimit::Outcome::LimitRaised { from, to } => { + debug!( + log, + "Raised soft open file descriptor resource limit"; + "old_limit" => from, + "new_limit" => to + ); + } + fdlimit::Outcome::Unsupported => { + debug!( + log, + "Raising soft open file descriptor resource limit is not supported" + ); + } + }; + info!( log, "Starting validator client";