From a9644f4ed129c9293b69c3ba22e4ef83a95841d5 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 21 Aug 2023 05:02:32 +0000 Subject: [PATCH] Remove checkpoint alignment requirements and enable historic state pruning (#4610) Closes #3210 Closes #3211 - Checkpoint sync from the latest finalized state regardless of its alignment. - Add the `block_root` to the database's split point. This is _only_ added to the in-memory split in order to avoid a schema migration. See `load_split`. - Add a new method to the DB called `get_advanced_state`, which looks up a state _by block root_, with a `state_root` as fallback. Using this method prevents accidental accesses of the split's unadvanced state, which does not exist in the hot DB and is not guaranteed to exist in the freezer DB at all. Previously Lighthouse would look up this state _from the freezer DB_, even if it was required for block/attestation processing, which was suboptimal. - Replace several state look-ups in block and attestation processing with `get_advanced_state` so that they can't hit the split block's unadvanced state. - Do not store any states in the freezer database by default. All states will be deleted upon being evicted from the hot database unless `--reconstruct-historic-states` is set. The anchor info which was previously used for checkpoint sync is used to implement this, including when syncing from genesis. Needs further testing. I want to stress-test the pruned database under Hydra. The `get_advanced_state` method is intended to become more relevant over time: `tree-states` includes an identically named method that returns advanced states from its in-memory cache. Co-authored-by: realbigsean --- beacon_node/beacon_chain/src/beacon_chain.rs | 13 +-- .../src/beacon_fork_choice_store.rs | 12 ++- .../beacon_chain/src/block_verification.rs | 17 ++-- beacon_node/beacon_chain/src/builder.rs | 86 +++++++++++-------- .../beacon_chain/src/canonical_head.rs | 23 +++-- beacon_node/beacon_chain/src/migrate.rs | 8 +- .../tests/attestation_verification.rs | 10 ++- .../tests/payload_invalidation.rs | 6 +- beacon_node/beacon_chain/tests/tests.rs | 6 +- beacon_node/client/Cargo.toml | 2 +- beacon_node/client/src/builder.rs | 76 ++++------------ beacon_node/http_api/tests/tests.rs | 21 ++++- .../gossip_methods.rs | 2 +- beacon_node/store/src/metadata.rs | 3 + book/src/checkpoint-sync.md | 23 ++--- consensus/fork_choice/src/fork_choice.rs | 3 +- .../src/fork_choice_test_definition.rs | 1 + .../src/proto_array_fork_choice.rs | 5 +- .../src/per_slot_processing.rs | 2 +- testing/ef_tests/src/cases/fork_choice.rs | 6 +- testing/node_test_rig/src/lib.rs | 3 + watch/tests/tests.rs | 9 +- 22 files changed, 190 insertions(+), 147 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index db238363d96..c8681f85d87 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -4305,6 +4305,7 @@ impl BeaconChain { self.log, "Produced block on state"; "block_size" => block_size, + "slot" => block.slot(), ); metrics::observe(&metrics::BLOCK_SIZE, block_size as f64); @@ -5178,14 +5179,16 @@ impl BeaconChain { let (mut state, state_root) = if let Some((state, state_root)) = head_state_opt { (state, state_root) } else { - let state_root = head_block.state_root; - let state = self + let block_state_root = head_block.state_root; + let max_slot = shuffling_epoch.start_slot(T::EthSpec::slots_per_epoch()); + let (state_root, state) = self .store .get_inconsistent_state_for_attestation_verification_only( - &state_root, - Some(head_block.slot), + &head_block_root, + max_slot, + block_state_root, )? - .ok_or(Error::MissingBeaconState(head_block.state_root))?; + .ok_or(Error::MissingBeaconState(block_state_root))?; (state, state_root) }; diff --git a/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs b/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs index 9b2edbd8b5d..2a42b49b422 100644 --- a/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs +++ b/beacon_node/beacon_chain/src/beacon_fork_choice_store.rs @@ -321,9 +321,17 @@ where .deconstruct() .0; - let state = self + let max_slot = self + .justified_checkpoint + .epoch + .start_slot(E::slots_per_epoch()); + let (_, state) = self .store - .get_state(&justified_block.state_root(), Some(justified_block.slot())) + .get_advanced_hot_state( + self.justified_checkpoint.root, + max_slot, + justified_block.state_root(), + ) .map_err(Error::FailedToReadState)? .ok_or_else(|| Error::MissingState(justified_block.state_root()))?; diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 5602b03c11f..cd470e6e891 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -1244,7 +1244,7 @@ impl ExecutionPendingBlock { // Perform a sanity check on the pre-state. let parent_slot = parent.beacon_block.slot(); - if state.slot() < parent_slot || state.slot() > parent_slot + 1 { + if state.slot() < parent_slot || state.slot() > block.slot() { return Err(BeaconChainError::BadPreState { parent_root: parent.beacon_block_root, parent_slot, @@ -1844,13 +1844,18 @@ fn load_parent( BlockError::from(BeaconChainError::MissingBeaconBlock(block.parent_root())) })?; - // Load the parent blocks state from the database, returning an error if it is not found. + // Load the parent block's state from the database, returning an error if it is not found. // It is an error because if we know the parent block we should also know the parent state. - let parent_state_root = parent_block.state_root(); - let parent_state = chain - .get_state(&parent_state_root, Some(parent_block.slot()))? + // Retrieve any state that is advanced through to at most `block.slot()`: this is + // particularly important if `block` descends from the finalized/split block, but at a slot + // prior to the finalized slot (which is invalid and inaccessible in our DB schema). + let (parent_state_root, parent_state) = chain + .store + .get_advanced_hot_state(root, block.slot(), parent_block.state_root())? .ok_or_else(|| { - BeaconChainError::DBInconsistent(format!("Missing state {:?}", parent_state_root)) + BeaconChainError::DBInconsistent( + format!("Missing state for parent block {root:?}",), + ) })?; metrics::inc_counter(&metrics::BLOCK_PROCESSING_SNAPSHOT_CACHE_MISSES); diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 916601cc0bb..7747e534caa 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -26,8 +26,9 @@ use operation_pool::{OperationPool, PersistedOperationPool}; use parking_lot::RwLock; use proto_array::{DisallowedReOrgOffsets, ReOrgThreshold}; use slasher::Slasher; -use slog::{crit, error, info, Logger}; +use slog::{crit, debug, error, info, Logger}; use slot_clock::{SlotClock, TestingSlotClock}; +use state_processing::per_slot_processing; use std::marker::PhantomData; use std::sync::Arc; use std::time::Duration; @@ -291,7 +292,7 @@ where let genesis_state = store .get_state(&genesis_block.state_root(), Some(genesis_block.slot())) .map_err(|e| descriptive_db_error("genesis state", &e))? - .ok_or("Genesis block not found in store")?; + .ok_or("Genesis state not found in store")?; self.genesis_time = Some(genesis_state.genesis_time()); @@ -386,6 +387,16 @@ where let (genesis, updated_builder) = self.set_genesis_state(beacon_state)?; self = updated_builder; + // Stage the database's metadata fields for atomic storage when `build` is called. + // Since v4.4.0 we will set the anchor with a dummy state upper limit in order to prevent + // historic states from being retained (unless `--reconstruct-historic-states` is set). + let retain_historic_states = self.chain_config.reconstruct_historic_states; + self.pending_io_batch.push( + store + .init_anchor_info(genesis.beacon_block.message(), retain_historic_states) + .map_err(|e| format!("Failed to initialize genesis anchor: {:?}", e))?, + ); + let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store, &genesis) .map_err(|e| format!("Unable to initialize fork choice store: {e:?}"))?; let current_slot = None; @@ -412,30 +423,28 @@ where weak_subj_block: SignedBeaconBlock, genesis_state: BeaconState, ) -> Result { - let store = self.store.clone().ok_or("genesis_state requires a store")?; - - let weak_subj_slot = weak_subj_state.slot(); - let weak_subj_block_root = weak_subj_block.canonical_root(); - let weak_subj_state_root = weak_subj_block.state_root(); - - // Check that the given block lies on an epoch boundary. Due to the database only storing - // full states on epoch boundaries and at restore points it would be difficult to support - // starting from a mid-epoch state. - if weak_subj_slot % TEthSpec::slots_per_epoch() != 0 { - return Err(format!( - "Checkpoint block at slot {} is not aligned to epoch start. \ - Please supply an aligned checkpoint with block.slot % 32 == 0", - weak_subj_block.slot(), - )); - } + let store = self + .store + .clone() + .ok_or("weak_subjectivity_state requires a store")?; + let log = self + .log + .as_ref() + .ok_or("weak_subjectivity_state requires a log")?; - // Check that the block and state have consistent slots and state roots. - if weak_subj_state.slot() != weak_subj_block.slot() { - return Err(format!( - "Slot of snapshot block ({}) does not match snapshot state ({})", - weak_subj_block.slot(), - weak_subj_state.slot(), - )); + // Ensure the state is advanced to an epoch boundary. + let slots_per_epoch = TEthSpec::slots_per_epoch(); + if weak_subj_state.slot() % slots_per_epoch != 0 { + debug!( + log, + "Advancing checkpoint state to boundary"; + "state_slot" => weak_subj_state.slot(), + "block_slot" => weak_subj_block.slot(), + ); + while weak_subj_state.slot() % slots_per_epoch != 0 { + per_slot_processing(&mut weak_subj_state, None, &self.spec) + .map_err(|e| format!("Error advancing state: {e:?}"))?; + } } // Prime all caches before storing the state in the database and computing the tree hash @@ -443,15 +452,19 @@ where weak_subj_state .build_caches(&self.spec) .map_err(|e| format!("Error building caches on checkpoint state: {e:?}"))?; - - let computed_state_root = weak_subj_state + let weak_subj_state_root = weak_subj_state .update_tree_hash_cache() .map_err(|e| format!("Error computing checkpoint state root: {:?}", e))?; - if weak_subj_state_root != computed_state_root { + let weak_subj_slot = weak_subj_state.slot(); + let weak_subj_block_root = weak_subj_block.canonical_root(); + + // Validate the state's `latest_block_header` against the checkpoint block. + let state_latest_block_root = weak_subj_state.get_latest_block_root(weak_subj_state_root); + if weak_subj_block_root != state_latest_block_root { return Err(format!( - "Snapshot state root does not match block, expected: {:?}, got: {:?}", - weak_subj_state_root, computed_state_root + "Snapshot state's most recent block root does not match block, expected: {:?}, got: {:?}", + weak_subj_block_root, state_latest_block_root )); } @@ -468,7 +481,7 @@ where // Set the store's split point *before* storing genesis so that genesis is stored // immediately in the freezer DB. - store.set_split(weak_subj_slot, weak_subj_state_root); + store.set_split(weak_subj_slot, weak_subj_state_root, weak_subj_block_root); let (_, updated_builder) = self.set_genesis_state(genesis_state)?; self = updated_builder; @@ -484,10 +497,11 @@ where // Stage the database's metadata fields for atomic storage when `build` is called. // This prevents the database from restarting in an inconsistent state if the anchor // info or split point is written before the `PersistedBeaconChain`. + let retain_historic_states = self.chain_config.reconstruct_historic_states; self.pending_io_batch.push(store.store_split_in_batch()); self.pending_io_batch.push( store - .init_anchor_info(weak_subj_block.message()) + .init_anchor_info(weak_subj_block.message(), retain_historic_states) .map_err(|e| format!("Failed to initialize anchor info: {:?}", e))?, ); @@ -507,13 +521,12 @@ where let fc_store = BeaconForkChoiceStore::get_forkchoice_store(store, &snapshot) .map_err(|e| format!("Unable to initialize fork choice store: {e:?}"))?; - let current_slot = Some(snapshot.beacon_block.slot()); let fork_choice = ForkChoice::from_anchor( fc_store, snapshot.beacon_block_root, &snapshot.beacon_block, &snapshot.beacon_state, - current_slot, + Some(weak_subj_slot), &self.spec, ) .map_err(|e| format!("Unable to initialize ForkChoice: {:?}", e))?; @@ -689,9 +702,8 @@ where Err(e) => return Err(descriptive_db_error("head block", &e)), }; - let head_state_root = head_block.state_root(); - let head_state = store - .get_state(&head_state_root, Some(head_block.slot())) + let (_head_state_root, head_state) = store + .get_advanced_hot_state(head_block_root, current_slot, head_block.state_root()) .map_err(|e| descriptive_db_error("head state", &e))? .ok_or("Head state not found in store")?; diff --git a/beacon_node/beacon_chain/src/canonical_head.rs b/beacon_node/beacon_chain/src/canonical_head.rs index 1cc4c3ea065..eeb25027dfb 100644 --- a/beacon_node/beacon_chain/src/canonical_head.rs +++ b/beacon_node/beacon_chain/src/canonical_head.rs @@ -47,7 +47,8 @@ use crate::{ }; use eth2::types::{EventKind, SseChainReorg, SseFinalizedCheckpoint, SseHead, SseLateHead}; use fork_choice::{ - ExecutionStatus, ForkChoiceView, ForkchoiceUpdateParameters, ProtoBlock, ResetPayloadStatuses, + ExecutionStatus, ForkChoiceStore, ForkChoiceView, ForkchoiceUpdateParameters, ProtoBlock, + ResetPayloadStatuses, }; use itertools::process_results; use parking_lot::{Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard}; @@ -298,10 +299,10 @@ impl CanonicalHead { let beacon_block = store .get_full_block(&beacon_block_root)? .ok_or(Error::MissingBeaconBlock(beacon_block_root))?; - let beacon_state_root = beacon_block.state_root(); - let beacon_state = store - .get_state(&beacon_state_root, Some(beacon_block.slot()))? - .ok_or(Error::MissingBeaconState(beacon_state_root))?; + let current_slot = fork_choice.fc_store().get_current_slot(); + let (_, beacon_state) = store + .get_advanced_hot_state(beacon_block_root, current_slot, beacon_block.state_root())? + .ok_or(Error::MissingBeaconState(beacon_block.state_root()))?; let snapshot = BeaconSnapshot { beacon_block_root, @@ -669,10 +670,14 @@ impl BeaconChain { .get_full_block(&new_view.head_block_root)? .ok_or(Error::MissingBeaconBlock(new_view.head_block_root))?; - let beacon_state_root = beacon_block.state_root(); - let beacon_state: BeaconState = self - .get_state(&beacon_state_root, Some(beacon_block.slot()))? - .ok_or(Error::MissingBeaconState(beacon_state_root))?; + let (_, beacon_state) = self + .store + .get_advanced_hot_state( + new_view.head_block_root, + current_slot, + beacon_block.state_root(), + )? + .ok_or(Error::MissingBeaconState(beacon_block.state_root()))?; Ok(BeaconSnapshot { beacon_block: Arc::new(beacon_block), diff --git a/beacon_node/beacon_chain/src/migrate.rs b/beacon_node/beacon_chain/src/migrate.rs index 4e7af30cb84..00c58679c42 100644 --- a/beacon_node/beacon_chain/src/migrate.rs +++ b/beacon_node/beacon_chain/src/migrate.rs @@ -289,6 +289,7 @@ impl, Cold: ItemStore> BackgroundMigrator state, @@ -342,7 +343,12 @@ impl, Cold: ItemStore> BackgroundMigrator {} Err(Error::HotColdDBError(HotColdDBError::FreezeSlotUnaligned(slot))) => { debug!( diff --git a/beacon_node/beacon_chain/tests/attestation_verification.rs b/beacon_node/beacon_chain/tests/attestation_verification.rs index 5cea51090b9..7878fd14aa0 100644 --- a/beacon_node/beacon_chain/tests/attestation_verification.rs +++ b/beacon_node/beacon_chain/tests/attestation_verification.rs @@ -9,7 +9,7 @@ use beacon_chain::{ test_utils::{ test_spec, AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType, }, - BeaconChain, BeaconChainError, BeaconChainTypes, WhenSlotSkipped, + BeaconChain, BeaconChainError, BeaconChainTypes, ChainConfig, WhenSlotSkipped, }; use genesis::{interop_genesis_state, DEFAULT_ETH1_BLOCK_HASH}; use int_to_bytes::int_to_bytes32; @@ -47,6 +47,10 @@ fn get_harness(validator_count: usize) -> BeaconChainHarness BeaconChainHarness> { let harness = BeaconChainHarness::builder(MinimalEthSpec) .default_spec() + .chain_config(ChainConfig { + reconstruct_historic_states: true, + ..ChainConfig::default() + }) .keypairs(KEYPAIRS[0..validator_count].to_vec()) .fresh_ephemeral_store() .mock_execution_layer() diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 72b6a6c7d47..0b517930fb9 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -6,11 +6,11 @@ edition = "2021" [dev-dependencies] serde_yaml = "0.8.13" -state_processing = { path = "../../consensus/state_processing" } operation_pool = { path = "../operation_pool" } tokio = "1.14.0" [dependencies] +state_processing = { path = "../../consensus/state_processing" } beacon_chain = { path = "../beacon_chain" } store = { path = "../store" } network = { path = "../network" } diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index fc3e5f5cefe..6955c369f67 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -317,7 +317,6 @@ where config.chain.checkpoint_sync_url_timeout, )), ); - let slots_per_epoch = TEthSpec::slots_per_epoch(); let deposit_snapshot = if config.sync_eth1_chain { // We want to fetch deposit snapshot before fetching the finalized beacon state to @@ -364,10 +363,23 @@ where None }; - debug!(context.log(), "Downloading finalized block"); - // Find a suitable finalized block on an epoch boundary. - let mut block = remote - .get_beacon_blocks_ssz::(BlockId::Finalized, &spec) + debug!( + context.log(), + "Downloading finalized state"; + ); + let state = remote + .get_debug_beacon_states_ssz::(StateId::Finalized, &spec) + .await + .map_err(|e| format!("Error loading checkpoint state from remote: {:?}", e))? + .ok_or_else(|| "Checkpoint state missing from remote".to_string())?; + + debug!(context.log(), "Downloaded finalized state"; "slot" => ?state.slot()); + + let finalized_block_slot = state.latest_block_header().slot; + + debug!(context.log(), "Downloading finalized block"; "block_slot" => ?finalized_block_slot); + let block = remote + .get_beacon_blocks_ssz::(BlockId::Slot(finalized_block_slot), &spec) .await .map_err(|e| match e { ApiError::InvalidSsz(e) => format!( @@ -381,65 +393,15 @@ where debug!(context.log(), "Downloaded finalized block"); - let mut block_slot = block.slot(); - - while block.slot() % slots_per_epoch != 0 { - block_slot = (block_slot / slots_per_epoch - 1) * slots_per_epoch; - - debug!( - context.log(), - "Searching for aligned checkpoint block"; - "block_slot" => block_slot - ); - - if let Some(found_block) = remote - .get_beacon_blocks_ssz::(BlockId::Slot(block_slot), &spec) - .await - .map_err(|e| { - format!("Error fetching block at slot {}: {:?}", block_slot, e) - })? - { - block = found_block; - } - } - - debug!( - context.log(), - "Downloaded aligned finalized block"; - "block_root" => ?block.canonical_root(), - "block_slot" => block.slot(), - ); - - let state_root = block.state_root(); - debug!( - context.log(), - "Downloading finalized state"; - "state_root" => ?state_root - ); - let state = remote - .get_debug_beacon_states_ssz::(StateId::Root(state_root), &spec) - .await - .map_err(|e| { - format!( - "Error loading checkpoint state from remote {:?}: {:?}", - state_root, e - ) - })? - .ok_or_else(|| { - format!("Checkpoint state missing from remote: {:?}", state_root) - })?; - - debug!(context.log(), "Downloaded finalized state"); - let genesis_state = BeaconState::from_ssz_bytes(&genesis_state_bytes, &spec) .map_err(|e| format!("Unable to parse genesis state SSZ: {:?}", e))?; info!( context.log(), "Loaded checkpoint block and state"; - "slot" => block.slot(), + "block_slot" => block.slot(), + "state_slot" => state.slot(), "block_root" => ?block.canonical_root(), - "state_root" => ?state_root, ); let service = diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 3ae495378e3..3c72441c02f 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -1,7 +1,7 @@ use beacon_chain::test_utils::RelativeSyncCommittee; use beacon_chain::{ test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType}, - BeaconChain, StateSkipConfig, WhenSlotSkipped, + BeaconChain, ChainConfig, StateSkipConfig, WhenSlotSkipped, }; use environment::null_logger; use eth2::{ @@ -77,6 +77,7 @@ struct ApiTester { struct ApiTesterConfig { spec: ChainSpec, + retain_historic_states: bool, builder_threshold: Option, } @@ -86,11 +87,19 @@ impl Default for ApiTesterConfig { spec.shard_committee_period = 2; Self { spec, + retain_historic_states: false, builder_threshold: None, } } } +impl ApiTesterConfig { + fn retain_historic_states(mut self) -> Self { + self.retain_historic_states = true; + self + } +} + impl ApiTester { pub async fn new() -> Self { // This allows for testing voluntary exits without building out a massive chain. @@ -118,6 +127,10 @@ impl ApiTester { let harness = Arc::new( BeaconChainHarness::builder(MainnetEthSpec) .spec(spec.clone()) + .chain_config(ChainConfig { + reconstruct_historic_states: config.retain_historic_states, + ..ChainConfig::default() + }) .logger(logging::test_logger()) .deterministic_keypairs(VALIDATOR_COUNT) .fresh_ephemeral_store() @@ -375,6 +388,7 @@ impl ApiTester { pub async fn new_mev_tester_no_builder_threshold() -> Self { let mut config = ApiTesterConfig { builder_threshold: Some(0), + retain_historic_states: false, spec: E::default_spec(), }; config.spec.altair_fork_epoch = Some(Epoch::new(0)); @@ -4705,7 +4719,7 @@ async fn get_validator_duties_attester_with_skip_slots() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn get_validator_duties_proposer() { - ApiTester::new() + ApiTester::new_from_config(ApiTesterConfig::default().retain_historic_states()) .await .test_get_validator_duties_proposer() .await; @@ -4713,7 +4727,7 @@ async fn get_validator_duties_proposer() { #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn get_validator_duties_proposer_with_skip_slots() { - ApiTester::new() + ApiTester::new_from_config(ApiTesterConfig::default().retain_historic_states()) .await .skip_slots(E::slots_per_epoch() * 2) .test_get_validator_duties_proposer() @@ -5045,6 +5059,7 @@ async fn builder_payload_chosen_by_profit() { async fn builder_works_post_capella() { let mut config = ApiTesterConfig { builder_threshold: Some(0), + retain_historic_states: false, spec: E::default_spec(), }; config.spec.altair_fork_epoch = Some(Epoch::new(0)); diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 3cba174e559..4e0cac134ff 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -1808,7 +1808,7 @@ impl Worker { ); } AttnError::BeaconChainError(BeaconChainError::DBError(Error::HotColdDBError( - HotColdDBError::AttestationStateIsFinalized { .. }, + HotColdDBError::FinalizedStateNotInHotDatabase { .. }, ))) => { debug!(self.log, "Attestation for finalized state"; "peer_id" => % peer_id); self.propagate_validation_result(message_id, peer_id, MessageAcceptance::Ignore); diff --git a/beacon_node/store/src/metadata.rs b/beacon_node/store/src/metadata.rs index 95e646ff51e..6405aff1bce 100644 --- a/beacon_node/store/src/metadata.rs +++ b/beacon_node/store/src/metadata.rs @@ -17,6 +17,9 @@ pub const COMPACTION_TIMESTAMP_KEY: Hash256 = Hash256::repeat_byte(4); pub const ANCHOR_INFO_KEY: Hash256 = Hash256::repeat_byte(5); pub const BLOB_INFO_KEY: Hash256 = Hash256::repeat_byte(6); +/// State upper limit value used to indicate that a node is not storing historic states. +pub const STATE_UPPER_LIMIT_NO_RETAIN: Slot = Slot::new(u64::MAX); + #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] pub struct SchemaVersion(pub u64); diff --git a/book/src/checkpoint-sync.md b/book/src/checkpoint-sync.md index 57883828947..0c375a5f009 100644 --- a/book/src/checkpoint-sync.md +++ b/book/src/checkpoint-sync.md @@ -75,7 +75,7 @@ Once backfill is complete, a `INFO Historical block download complete` log will > Note: Since [v4.1.0](https://github.com/sigp/lighthouse/releases/tag/v4.1.0), Lighthouse implements rate-limited backfilling to mitigate validator performance issues after a recent checkpoint sync. This means that the speed at which historical blocks are downloaded is limited, typically to less than 20 slots/sec. This will not affect validator performance. However, if you would still prefer to sync the chain as fast as possible, you can add the flag `--disable-backfill-rate-limiting` to the beacon node. -> Note: Since [v4.2.0](https://github.com/sigp/lighthouse/releases/tag/v4.2.0), Lighthouse limits the backfill sync to only sync backwards to the weak subjectivity point (approximately 5 months). This will help to save disk space. However, if you would like to sync back to the genesis, you can add the flag `--genesis-backfill` to the beacon node. +> Note: Since [v4.2.0](https://github.com/sigp/lighthouse/releases/tag/v4.2.0), Lighthouse limits the backfill sync to only sync backwards to the weak subjectivity point (approximately 5 months). This will help to save disk space. However, if you would like to sync back to the genesis, you can add the flag `--genesis-backfill` to the beacon node. ## FAQ @@ -116,8 +116,9 @@ states: database. Additionally, the genesis block is always available. * `state_lower_limit`: All states with slots _less than or equal to_ this value are available in the database. The minimum value is 0, indicating that the genesis state is always available. -* `state_upper_limit`: All states with slots _greater than or equal to_ this value are available - in the database. +* `state_upper_limit`: All states with slots _greater than or equal to_ `min(split.slot, + state_upper_limit)` are available in the database. In the case where the `state_upper_limit` is + higher than the `split.slot`, this means states are not being written to the freezer database. Reconstruction runs from the state lower limit to the upper limit, narrowing the window of unavailable states as it goes. It will log messages like the following to show its progress: @@ -153,18 +154,8 @@ To manually specify a checkpoint use the following two flags: * `--checkpoint-state`: accepts an SSZ-encoded `BeaconState` blob * `--checkpoint-block`: accepts an SSZ-encoded `SignedBeaconBlock` blob -_Both_ the state and block must be provided and **must** adhere to the [Alignment -Requirements](#alignment-requirements) described below. - -### Alignment Requirements - -* The block must be a finalized block from an epoch boundary, i.e. `block.slot() % 32 == 0`. -* The state must be the state corresponding to `block` with `state.slot() == block.slot()` - and `state.hash_tree_root() == block.state_root()`. - -These requirements are imposed to align with Lighthouse's database schema, and notably exclude -finalized blocks from skipped slots. You can avoid alignment issues by using -[Automatic Checkpoint Sync](#automatic-checkpoint-sync), which will search for a suitable block -and state pair. +_Both_ the state and block must be provided and the state **must** match the block. The +state may be from the same slot as the block (unadvanced), or advanced to an epoch boundary, +in which case it will be assumed to be finalized at that epoch. [weak-subj]: https://blog.ethereum.org/2014/11/25/proof-stake-learned-love-weak-subjectivity/ diff --git a/consensus/fork_choice/src/fork_choice.rs b/consensus/fork_choice/src/fork_choice.rs index be135918040..73b44809620 100644 --- a/consensus/fork_choice/src/fork_choice.rs +++ b/consensus/fork_choice/src/fork_choice.rs @@ -350,7 +350,7 @@ where spec: &ChainSpec, ) -> Result> { // Sanity check: the anchor must lie on an epoch boundary. - if anchor_block.slot() % E::slots_per_epoch() != 0 { + if anchor_state.slot() % E::slots_per_epoch() != 0 { return Err(Error::InvalidAnchor { block_slot: anchor_block.slot(), state_slot: anchor_state.slot(), @@ -386,6 +386,7 @@ where let current_slot = current_slot.unwrap_or_else(|| fc_store.get_current_slot()); let proto_array = ProtoArrayForkChoice::new::( + current_slot, finalized_block_slot, finalized_block_state_root, *fc_store.justified_checkpoint(), diff --git a/consensus/proto_array/src/fork_choice_test_definition.rs b/consensus/proto_array/src/fork_choice_test_definition.rs index 157f072ad37..98d43e4850c 100644 --- a/consensus/proto_array/src/fork_choice_test_definition.rs +++ b/consensus/proto_array/src/fork_choice_test_definition.rs @@ -80,6 +80,7 @@ impl ForkChoiceTestDefinition { let junk_shuffling_id = AttestationShufflingId::from_components(Epoch::new(0), Hash256::zero()); let mut fork_choice = ProtoArrayForkChoice::new::( + self.finalized_block_slot, self.finalized_block_slot, Hash256::zero(), self.justified_checkpoint, diff --git a/consensus/proto_array/src/proto_array_fork_choice.rs b/consensus/proto_array/src/proto_array_fork_choice.rs index fe831b3c357..5911e50fcdc 100644 --- a/consensus/proto_array/src/proto_array_fork_choice.rs +++ b/consensus/proto_array/src/proto_array_fork_choice.rs @@ -345,6 +345,7 @@ pub struct ProtoArrayForkChoice { impl ProtoArrayForkChoice { #[allow(clippy::too_many_arguments)] pub fn new( + current_slot: Slot, finalized_block_slot: Slot, finalized_block_state_root: Hash256, justified_checkpoint: Checkpoint, @@ -380,7 +381,7 @@ impl ProtoArrayForkChoice { }; proto_array - .on_block::(block, finalized_block_slot) + .on_block::(block, current_slot) .map_err(|e| format!("Failed to add finalized block to proto_array: {:?}", e))?; Ok(Self { @@ -983,6 +984,7 @@ mod test_compute_deltas { }; let mut fc = ProtoArrayForkChoice::new::( + genesis_slot, genesis_slot, state_root, genesis_checkpoint, @@ -1108,6 +1110,7 @@ mod test_compute_deltas { }; let mut fc = ProtoArrayForkChoice::new::( + genesis_slot, genesis_slot, junk_state_root, genesis_checkpoint, diff --git a/consensus/state_processing/src/per_slot_processing.rs b/consensus/state_processing/src/per_slot_processing.rs index 8d2600bb41e..3fa841517dd 100644 --- a/consensus/state_processing/src/per_slot_processing.rs +++ b/consensus/state_processing/src/per_slot_processing.rs @@ -23,7 +23,7 @@ impl From for Error { /// /// If the root of the supplied `state` is known, then it can be passed as `state_root`. If /// `state_root` is `None`, the root of `state` will be computed using a cached tree hash. -/// Providing the `state_root` makes this function several orders of magniude faster. +/// Providing the `state_root` makes this function several orders of magnitude faster. pub fn per_slot_processing( state: &mut BeaconState, state_root: Option, diff --git a/testing/ef_tests/src/cases/fork_choice.rs b/testing/ef_tests/src/cases/fork_choice.rs index 8c4a90044a6..f3afb5f7c59 100644 --- a/testing/ef_tests/src/cases/fork_choice.rs +++ b/testing/ef_tests/src/cases/fork_choice.rs @@ -7,7 +7,7 @@ use beacon_chain::{ obtain_indexed_attestation_and_committees_per_slot, VerifiedAttestation, }, test_utils::{BeaconChainHarness, EphemeralHarnessType}, - BeaconChainTypes, CachedHead, NotifyExecutionLayer, + BeaconChainTypes, CachedHead, ChainConfig, NotifyExecutionLayer, }; use execution_layer::{json_structures::JsonPayloadStatusV1Status, PayloadStatusV1}; use serde::Deserialize; @@ -303,6 +303,10 @@ impl Tester { let harness = BeaconChainHarness::>::builder(E::default()) .spec(spec.clone()) .keypairs(vec![]) + .chain_config(ChainConfig { + reconstruct_historic_states: true, + ..ChainConfig::default() + }) .genesis_state_ephemeral_store(case.anchor_state.clone()) .mock_execution_layer() .recalculate_fork_times_with_genesis(0) diff --git a/testing/node_test_rig/src/lib.rs b/testing/node_test_rig/src/lib.rs index 394f8558fae..3cd8205eb65 100644 --- a/testing/node_test_rig/src/lib.rs +++ b/testing/node_test_rig/src/lib.rs @@ -115,6 +115,9 @@ pub fn testing_client_config() -> ClientConfig { genesis_time: now, }; + // Simulator tests expect historic states to be available for post-run checks. + client_config.chain.reconstruct_historic_states = true; + // Specify a constant count of beacon processor workers. Having this number // too low can cause annoying HTTP timeouts, especially on Github runners // with 2 logical CPUs. diff --git a/watch/tests/tests.rs b/watch/tests/tests.rs index af1cde26b7a..9032b124ab8 100644 --- a/watch/tests/tests.rs +++ b/watch/tests/tests.rs @@ -1,8 +1,9 @@ #![recursion_limit = "256"] #![cfg(unix)] -use beacon_chain::test_utils::{ - AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType, +use beacon_chain::{ + test_utils::{AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType}, + ChainConfig, }; use eth2::{types::BlockId, BeaconNodeHttpClient, SensitiveUrl, Timeouts}; use http_api::test_utils::{create_api_server, ApiServer}; @@ -91,6 +92,10 @@ impl TesterBuilder { pub async fn new() -> TesterBuilder { let harness = BeaconChainHarness::builder(E::default()) .default_spec() + .chain_config(ChainConfig { + reconstruct_historic_states: true, + ..ChainConfig::default() + }) .deterministic_keypairs(VALIDATOR_COUNT) .fresh_ephemeral_store() .build();