From 40d412629ecfad250d6f29dc540b50ab4e399639 Mon Sep 17 00:00:00 2001 From: Lion - dapplion <35266934+dapplion@users.noreply.github.com> Date: Mon, 29 Apr 2024 23:30:13 +0900 Subject: [PATCH 1/3] Ignore gossip blob already imported (#5656) * Ignore gossip blob already imported --- .../network_beacon_processor/gossip_methods.rs | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs index 7b8826bd853..af7f3a53e56 100644 --- a/beacon_node/network/src/network_beacon_processor/gossip_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/gossip_methods.rs @@ -778,24 +778,28 @@ impl NetworkBeaconProcessor { "block_root" => %block_root, ); } + Err(BlockError::BlockIsAlreadyKnown(_)) => { + debug!( + self.log, + "Ignoring gossip blob already imported"; + "block_root" => ?block_root, + "blob_index" => blob_index, + ); + } Err(err) => { debug!( self.log, "Invalid gossip blob"; "outcome" => ?err, - "block root" => ?block_root, - "block slot" => blob_slot, - "blob index" => blob_index, + "block_root" => ?block_root, + "block_slot" => blob_slot, + "blob_index" => blob_index, ); self.gossip_penalize_peer( peer_id, PeerAction::MidToleranceError, "bad_gossip_blob_ssz", ); - trace!( - self.log, - "Invalid gossip blob ssz"; - ); } } } From c33edc82ebcf0d6db45d8c715cbb071e0d62fcd9 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Mon, 29 Apr 2024 16:45:54 -0400 Subject: [PATCH 2/3] Beta compiler fix (#5659) * fix beta compiler compilation * remove unused import * Revert "remove unused import" This reverts commit 0bef36b05bb146cfcc4a2e0b77ddbd8852e3bf9f. * Revert "fix beta compiler compilation" This reverts commit 23152cf4cc1ee627b76585a013d093f14e45ab14. * rename ununsed fields * allow dead code on some error variants * remove unused blob download queue * add back debug to backfill error * more allow dead code on errors --- .../network/src/sync/backfill_sync/mod.rs | 9 +++++---- .../network/src/sync/block_lookups/common.rs | 2 +- .../sync/block_lookups/single_block_lookup.rs | 10 +++------- beacon_node/network/src/sync/manager.rs | 16 ++++++++-------- common/logging/src/lib.rs | 8 ++++---- common/logging/src/tracing_logging_layer.rs | 4 ++-- validator_client/src/duties_service.rs | 11 ++++++----- validator_client/src/http_metrics/mod.rs | 4 ++-- 8 files changed, 31 insertions(+), 33 deletions(-) diff --git a/beacon_node/network/src/sync/backfill_sync/mod.rs b/beacon_node/network/src/sync/backfill_sync/mod.rs index 67fe871accf..9075bb15f08 100644 --- a/beacon_node/network/src/sync/backfill_sync/mod.rs +++ b/beacon_node/network/src/sync/backfill_sync/mod.rs @@ -87,16 +87,17 @@ pub enum ProcessResult { } /// The ways a backfill sync can fail. +// The info in the enum variants is displayed in logging, clippy thinks it's dead code. #[derive(Debug)] pub enum BackFillError { /// A batch failed to be downloaded. - BatchDownloadFailed(BatchId), + BatchDownloadFailed(#[allow(dead_code)] BatchId), /// A batch could not be processed. - BatchProcessingFailed(BatchId), + BatchProcessingFailed(#[allow(dead_code)] BatchId), /// A batch entered an invalid state. - BatchInvalidState(BatchId, String), + BatchInvalidState(#[allow(dead_code)] BatchId, #[allow(dead_code)] String), /// The sync algorithm entered an invalid state. - InvalidSyncState(String), + InvalidSyncState(#[allow(dead_code)] String), /// The chain became paused. Paused, } diff --git a/beacon_node/network/src/sync/block_lookups/common.rs b/beacon_node/network/src/sync/block_lookups/common.rs index 7193dd6e216..9d1af0cb813 100644 --- a/beacon_node/network/src/sync/block_lookups/common.rs +++ b/beacon_node/network/src/sync/block_lookups/common.rs @@ -245,7 +245,7 @@ impl RequestState for BlockRequestState { } } -impl RequestState for BlobRequestState { +impl RequestState for BlobRequestState { type RequestType = BlobsByRootSingleBlockRequest; type VerifiedResponseType = FixedBlobSidecarList; type ReconstructedResponseType = FixedBlobSidecarList; diff --git a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs index 077af7c3d19..49ef1dd15bf 100644 --- a/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs +++ b/beacon_node/network/src/sync/block_lookups/single_block_lookup.rs @@ -18,7 +18,6 @@ use std::fmt::Debug; use std::sync::Arc; use store::Hash256; use strum::IntoStaticStr; -use types::blob_sidecar::FixedBlobSidecarList; use types::EthSpec; #[derive(Debug, PartialEq, Eq, IntoStaticStr)] @@ -37,7 +36,7 @@ pub struct SingleBlockLookup { pub id: Id, pub lookup_type: LookupType, pub block_request_state: BlockRequestState, - pub blob_request_state: BlobRequestState, + pub blob_request_state: BlobRequestState, pub da_checker: Arc>, /// Only necessary for requests triggered by an `UnknownBlockParent` or `UnknownBlockParent` /// because any blocks or blobs without parents won't hit the data availability cache. @@ -304,24 +303,21 @@ impl SingleBlockLookup { } /// The state of the blob request component of a `SingleBlockLookup`. -pub struct BlobRequestState { +pub struct BlobRequestState { /// The latest picture of which blobs still need to be requested. This includes information /// from both block/blobs downloaded in the network layer and any blocks/blobs that exist in /// the data availability checker. pub requested_ids: MissingBlobs, pub block_root: Hash256, - /// Where we store blobs until we receive the stream terminator. - pub blob_download_queue: FixedBlobSidecarList, pub state: SingleLookupRequestState, } -impl BlobRequestState { +impl BlobRequestState { pub fn new(block_root: Hash256, peer_source: &[PeerId], is_deneb: bool) -> Self { let default_ids = MissingBlobs::new_without_block(block_root, is_deneb); Self { block_root, requested_ids: default_ids, - blob_download_queue: <_>::default(), state: SingleLookupRequestState::new(peer_source), } } diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index 9c17c6a1512..15b96c52b10 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -627,7 +627,7 @@ impl SyncManager { ), BlockProcessType::SingleBlob { id } => self .block_lookups - .single_block_component_processed::>( + .single_block_component_processed::( id, result, &mut self.network, @@ -908,7 +908,7 @@ impl SyncManager { Ok((blobs, seen_timestamp)) => match id.lookup_type { LookupType::Current => self .block_lookups - .single_lookup_response::>( + .single_lookup_response::( id, peer_id, blobs, @@ -917,7 +917,7 @@ impl SyncManager { ), LookupType::Parent => self .block_lookups - .parent_lookup_response::>( + .parent_lookup_response::( id, peer_id, blobs, @@ -929,20 +929,20 @@ impl SyncManager { Err(error) => match id.lookup_type { LookupType::Current => self .block_lookups - .single_block_lookup_failed::>( + .single_block_lookup_failed::( id, &peer_id, &mut self.network, error, ), - LookupType::Parent => self - .block_lookups - .parent_lookup_failed::>( + LookupType::Parent => { + self.block_lookups.parent_lookup_failed::( id, &peer_id, &mut self.network, error, - ), + ) + } }, } } diff --git a/common/logging/src/lib.rs b/common/logging/src/lib.rs index 3a5a5209b02..b0e1da00e97 100644 --- a/common/logging/src/lib.rs +++ b/common/logging/src/lib.rs @@ -256,14 +256,14 @@ pub fn create_tracing_layer(base_tracing_log_path: PathBuf) { return; }; - let (libp2p_non_blocking_writer, libp2p_guard) = NonBlocking::new(libp2p_writer); - let (discv5_non_blocking_writer, discv5_guard) = NonBlocking::new(discv5_writer); + let (libp2p_non_blocking_writer, _libp2p_guard) = NonBlocking::new(libp2p_writer); + let (discv5_non_blocking_writer, _discv5_guard) = NonBlocking::new(discv5_writer); let custom_layer = LoggingLayer { libp2p_non_blocking_writer, - libp2p_guard, + _libp2p_guard, discv5_non_blocking_writer, - discv5_guard, + _discv5_guard, }; if let Err(e) = tracing_subscriber::fmt() diff --git a/common/logging/src/tracing_logging_layer.rs b/common/logging/src/tracing_logging_layer.rs index aabb6ddd0c9..a9ddae828ae 100644 --- a/common/logging/src/tracing_logging_layer.rs +++ b/common/logging/src/tracing_logging_layer.rs @@ -7,9 +7,9 @@ use tracing_subscriber::Layer; pub struct LoggingLayer { pub libp2p_non_blocking_writer: NonBlocking, - pub libp2p_guard: WorkerGuard, + pub _libp2p_guard: WorkerGuard, pub discv5_non_blocking_writer: NonBlocking, - pub discv5_guard: WorkerGuard, + pub _discv5_guard: WorkerGuard, } impl Layer for LoggingLayer diff --git a/validator_client/src/duties_service.rs b/validator_client/src/duties_service.rs index b5b56943c4b..15e5c45e0d5 100644 --- a/validator_client/src/duties_service.rs +++ b/validator_client/src/duties_service.rs @@ -88,14 +88,15 @@ const _: () = assert!({ /// bringing in the entire crate. const _: () = assert!(ATTESTATION_SUBSCRIPTION_OFFSETS[0] > 2); +// The info in the enum variants is displayed in logging, clippy thinks it's dead code. #[derive(Debug)] pub enum Error { UnableToReadSlotClock, - FailedToDownloadAttesters(String), - FailedToProduceSelectionProof(ValidatorStoreError), - InvalidModulo(ArithError), - Arith(ArithError), - SyncDutiesNotFound(u64), + FailedToDownloadAttesters(#[allow(dead_code)] String), + FailedToProduceSelectionProof(#[allow(dead_code)] ValidatorStoreError), + InvalidModulo(#[allow(dead_code)] ArithError), + Arith(#[allow(dead_code)] ArithError), + SyncDutiesNotFound(#[allow(dead_code)] u64), } impl From for Error { diff --git a/validator_client/src/http_metrics/mod.rs b/validator_client/src/http_metrics/mod.rs index de6c06437b4..67cab2bdc37 100644 --- a/validator_client/src/http_metrics/mod.rs +++ b/validator_client/src/http_metrics/mod.rs @@ -17,8 +17,8 @@ use warp::{http::Response, Filter}; #[derive(Debug)] pub enum Error { - Warp(warp::Error), - Other(String), + Warp(#[allow(dead_code)] warp::Error), + Other(#[allow(dead_code)] String), } impl From for Error { From c8ffafb3f8e5022661020354d727b12b573b9e88 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Tue, 30 Apr 2024 00:55:23 +0200 Subject: [PATCH 3/3] fix(validator_client): raise soft fd limit (#4796) * fix(validator_client): raise soft fd limit * Merge branch 'unstable' of https://github.com/sigp/lighthouse into rkrasiuk/raise-vc-fdlimit * cargo lock * Merge branch 'unstable' of https://github.com/sigp/lighthouse into rkrasiuk/raise-vc-fdlimit --- Cargo.lock | 11 +++++++++++ validator_client/Cargo.toml | 1 + validator_client/src/lib.rs | 23 ++++++++++++++++++++++- 3 files changed, 34 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index cc38877fe76..85c89528d0c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2912,6 +2912,16 @@ dependencies = [ "bytes", ] +[[package]] +name = "fdlimit" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e182f7dbc2ef73d9ef67351c5fbbea084729c48362d3ce9dd44c28e32e277fe5" +dependencies = [ + "libc", + "thiserror", +] + [[package]] name = "ff" version = "0.12.1" @@ -8988,6 +8998,7 @@ dependencies = [ "eth2", "eth2_keystore", "ethereum_serde_utils", + "fdlimit", "filesystem", "futures", "hex", diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index d3dffc3d02e..0df687abec5 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -61,3 +61,4 @@ sysinfo = { workspace = true } system_health = { path = "../common/system_health" } logging = { workspace = true } strum = { workspace = true } +fdlimit = "0.3.0" diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 268c25cdf7d..377b0640483 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -47,7 +47,7 @@ use notifier::spawn_notifier; use parking_lot::RwLock; use preparation_service::{PreparationService, PreparationServiceBuilder}; use reqwest::Certificate; -use slog::{error, info, warn, Logger}; +use slog::{debug, error, info, warn, Logger}; use slot_clock::SlotClock; use slot_clock::SystemTimeSlotClock; use std::fs::File; @@ -121,6 +121,27 @@ impl ProductionValidatorClient { pub async fn new(context: RuntimeContext, config: Config) -> Result { let log = context.log().clone(); + // Attempt to raise soft fd limit. The behavior is OS specific: + // `linux` - raise soft fd limit to hard + // `macos` - raise soft fd limit to `min(kernel limit, hard fd limit)` + // `windows` & rest - noop + match fdlimit::raise_fd_limit().map_err(|e| format!("Unable to raise fd limit: {}", e))? { + fdlimit::Outcome::LimitRaised { from, to } => { + debug!( + log, + "Raised soft open file descriptor resource limit"; + "old_limit" => from, + "new_limit" => to + ); + } + fdlimit::Outcome::Unsupported => { + debug!( + log, + "Raising soft open file descriptor resource limit is not supported" + ); + } + }; + info!( log, "Starting validator client";