From b60304b19f1607c12d52e32356678cef6c7ba4e2 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Tue, 8 Aug 2023 23:30:15 +0000 Subject: [PATCH 1/5] Use `BeaconProcessor` for API requests (#4462) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Issue Addressed NA ## Proposed Changes Rather than spawning new tasks on the tokio executor to process each HTTP API request, send the tasks to the `BeaconProcessor`. This achieves: 1. Places a bound on how many concurrent requests are being served (i.e., how many we are actually trying to compute at one time). 1. Places a bound on how many requests can be awaiting a response at one time (i.e., starts dropping requests when we have too many queued). 1. Allows the BN prioritise HTTP requests with respect to messages coming from the P2P network (i.e., proiritise importing gossip blocks rather than serving API requests). Presently there are two levels of priorities: - `Priority::P0` - The beacon processor will prioritise these above everything other than importing new blocks. - Roughly all validator-sensitive endpoints. - `Priority::P1` - The beacon processor will prioritise practically all other P2P messages over these, except for historical backfill things. - Everything that's not `Priority::P0` The `--http-enable-beacon-processor false` flag can be supplied to revert back to the old behaviour of spawning new `tokio` tasks for each request: ``` --http-enable-beacon-processor The beacon processor is a scheduler which provides quality-of-service and DoS protection. When set to "true", HTTP API requests will queued and scheduled alongside other tasks. When set to "false", HTTP API responses will be executed immediately. [default: true] ``` ## New CLI Flags I added some other new CLI flags: ``` --beacon-processor-aggregate-batch-size Specifies the number of gossip aggregate attestations in a signature verification batch. Higher values may reduce CPU usage in a healthy network while lower values may increase CPU usage in an unhealthy or hostile network. [default: 64] --beacon-processor-attestation-batch-size Specifies the number of gossip attestations in a signature verification batch. Higher values may reduce CPU usage in a healthy network whilst lower values may increase CPU usage in an unhealthy or hostile network. [default: 64] --beacon-processor-max-workers Specifies the maximum concurrent tasks for the task scheduler. Increasing this value may increase resource consumption. Reducing the value may result in decreased resource usage and diminished performance. The default value is the number of logical CPU cores on the host. --beacon-processor-reprocess-queue-len Specifies the length of the queue for messages requiring delayed processing. Higher values may prevent messages from being dropped while lower values may help protect the node from becoming overwhelmed. [default: 12288] ``` I needed to add the max-workers flag since the "simulator" flavor tests started failing with HTTP timeouts on the test assertions. I believe they were failing because the Github runners only have 2 cores and there just weren't enough workers available to process our requests in time. I added the other flags since they seem fun to fiddle with. ## Additional Info I bumped the timeouts on the "simulator" flavor test from 4s to 8s. The prioritisation of consensus messages seems to be causing slower responses, I guess this is what we signed up for 🤷 The `validator/register` validator has some special handling because the relays have a bad habit of timing out on these calls. It seems like a waste of a `BeaconProcessor` worker to just wait for the builder API HTTP response, so we spawn a new `tokio` task to wait for a builder response. I've added an optimisation for the `GET beacon/states/{state_id}/validators/{validator_id}` endpoint in [efbabe3](https://github.com/sigp/lighthouse/pull/4462/commits/efbabe32521ed6eb3564764da4e507d26a1c4bd0). That's the endpoint the VC uses to resolve pubkeys to validator indices, and it's the endpoint that was causing us grief. Perhaps I should move that into a new PR, not sure. --- Cargo.lock | 5 + beacon_node/beacon_chain/src/chain_config.rs | 3 - beacon_node/beacon_processor/Cargo.toml | 4 +- beacon_node/beacon_processor/src/lib.rs | 127 +- beacon_node/beacon_processor/src/metrics.rs | 9 + beacon_node/client/src/builder.rs | 58 +- beacon_node/client/src/config.rs | 3 + beacon_node/http_api/Cargo.toml | 13 +- beacon_node/http_api/src/lib.rs | 2062 ++++++++++------- beacon_node/http_api/src/task_spawner.rs | 214 ++ beacon_node/http_api/src/test_utils.rs | 59 +- beacon_node/http_api/src/validator.rs | 21 + beacon_node/http_api/tests/tests.rs | 10 +- .../src/network_beacon_processor/mod.rs | 22 +- .../src/network_beacon_processor/tests.rs | 37 +- beacon_node/network/src/service/tests.rs | 21 +- beacon_node/src/cli.rs | 62 + beacon_node/src/config.rs | 28 +- beacon_node/src/lib.rs | 1 + lighthouse/Cargo.toml | 1 + lighthouse/tests/beacon_node.rs | 58 +- testing/node_test_rig/src/lib.rs | 7 +- watch/Cargo.toml | 1 + watch/tests/tests.rs | 12 +- 24 files changed, 1870 insertions(+), 968 deletions(-) create mode 100644 beacon_node/http_api/src/task_spawner.rs create mode 100644 beacon_node/http_api/src/validator.rs diff --git a/Cargo.lock b/Cargo.lock index a24087c3a06..233b3901ebf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -601,7 +601,9 @@ dependencies = [ "lighthouse_metrics", "lighthouse_network", "logging", + "num_cpus", "parking_lot 0.12.1", + "serde", "slog", "slot_clock", "strum", @@ -3266,6 +3268,7 @@ name = "http_api" version = "0.1.0" dependencies = [ "beacon_chain", + "beacon_processor", "bs58 0.4.0", "bytes", "directory", @@ -4320,6 +4323,7 @@ dependencies = [ "account_manager", "account_utils", "beacon_node", + "beacon_processor", "bls", "boot_node", "clap", @@ -8893,6 +8897,7 @@ dependencies = [ "serde", "serde_json", "serde_yaml", + "task_executor", "testcontainers", "tokio", "tokio-postgres", diff --git a/beacon_node/beacon_chain/src/chain_config.rs b/beacon_node/beacon_chain/src/chain_config.rs index efbc9905b7a..d1bddcf736a 100644 --- a/beacon_node/beacon_chain/src/chain_config.rs +++ b/beacon_node/beacon_chain/src/chain_config.rs @@ -79,8 +79,6 @@ pub struct ChainConfig { /// /// This is useful for block builders and testing. pub always_prepare_payload: bool, - /// Whether backfill sync processing should be rate-limited. - pub enable_backfill_rate_limiting: bool, /// Whether to use `ProgressiveBalancesCache` in unrealized FFG progression calculation. pub progressive_balances_mode: ProgressiveBalancesMode, /// Number of epochs between each migration of data from the hot database to the freezer. @@ -114,7 +112,6 @@ impl Default for ChainConfig { shuffling_cache_size: crate::shuffling_cache::DEFAULT_CACHE_SIZE, genesis_backfill: false, always_prepare_payload: false, - enable_backfill_rate_limiting: true, progressive_balances_mode: ProgressiveBalancesMode::Checked, epochs_per_migration: crate::migrate::DEFAULT_EPOCHS_PER_MIGRATION, } diff --git a/beacon_node/beacon_processor/Cargo.toml b/beacon_node/beacon_processor/Cargo.toml index 5c5200e101e..c626441bb71 100644 --- a/beacon_node/beacon_processor/Cargo.toml +++ b/beacon_node/beacon_processor/Cargo.toml @@ -21,4 +21,6 @@ types = { path = "../../consensus/types" } ethereum_ssz = "0.5.0" lazy_static = "1.4.0" lighthouse_metrics = { path = "../../common/lighthouse_metrics" } -parking_lot = "0.12.0" \ No newline at end of file +parking_lot = "0.12.0" +num_cpus = "1.13.0" +serde = { version = "1.0.116", features = ["derive"] } \ No newline at end of file diff --git a/beacon_node/beacon_processor/src/lib.rs b/beacon_node/beacon_processor/src/lib.rs index 297c4868db7..bf5d8bced46 100644 --- a/beacon_node/beacon_processor/src/lib.rs +++ b/beacon_node/beacon_processor/src/lib.rs @@ -48,6 +48,7 @@ use lighthouse_network::NetworkGlobals; use lighthouse_network::{MessageId, PeerId}; use logging::TimeLatch; use parking_lot::Mutex; +use serde::{Deserialize, Serialize}; use slog::{crit, debug, error, trace, warn, Logger}; use slot_clock::SlotClock; use std::cmp; @@ -70,7 +71,7 @@ pub mod work_reprocessing_queue; /// The maximum size of the channel for work events to the `BeaconProcessor`. /// /// Setting this too low will cause consensus messages to be dropped. -pub const MAX_WORK_EVENT_QUEUE_LEN: usize = 16_384; +const DEFAULT_MAX_WORK_EVENT_QUEUE_LEN: usize = 16_384; /// The maximum size of the channel for idle events to the `BeaconProcessor`. /// @@ -79,7 +80,7 @@ pub const MAX_WORK_EVENT_QUEUE_LEN: usize = 16_384; const MAX_IDLE_QUEUE_LEN: usize = 16_384; /// The maximum size of the channel for re-processing work events. -pub const MAX_SCHEDULED_WORK_QUEUE_LEN: usize = 3 * MAX_WORK_EVENT_QUEUE_LEN / 4; +const DEFAULT_MAX_SCHEDULED_WORK_QUEUE_LEN: usize = 3 * DEFAULT_MAX_WORK_EVENT_QUEUE_LEN / 4; /// The maximum number of queued `Attestation` objects that will be stored before we start dropping /// them. @@ -167,6 +168,14 @@ const MAX_BLS_TO_EXECUTION_CHANGE_QUEUE_LEN: usize = 16_384; /// will be stored before we start dropping them. const MAX_LIGHT_CLIENT_BOOTSTRAP_QUEUE_LEN: usize = 1_024; +/// The maximum number of priority-0 (highest priority) messages that will be queued before +/// they begin to be dropped. +const MAX_API_REQUEST_P0_QUEUE_LEN: usize = 1_024; + +/// The maximum number of priority-1 (second-highest priority) messages that will be queued before +/// they begin to be dropped. +const MAX_API_REQUEST_P1_QUEUE_LEN: usize = 1_024; + /// The name of the manager tokio task. const MANAGER_TASK_NAME: &str = "beacon_processor_manager"; @@ -184,8 +193,8 @@ const WORKER_TASK_NAME: &str = "beacon_processor_worker"; /// Poisoning occurs when an invalid signature is included in a batch of attestations. A single /// invalid signature causes the entire batch to fail. When a batch fails, we fall-back to /// individually verifying each attestation signature. -const MAX_GOSSIP_ATTESTATION_BATCH_SIZE: usize = 64; -const MAX_GOSSIP_AGGREGATE_BATCH_SIZE: usize = 64; +const DEFAULT_MAX_GOSSIP_ATTESTATION_BATCH_SIZE: usize = 64; +const DEFAULT_MAX_GOSSIP_AGGREGATE_BATCH_SIZE: usize = 64; /// Unique IDs used for metrics and testing. pub const WORKER_FREED: &str = "worker_freed"; @@ -215,6 +224,61 @@ pub const UNKNOWN_BLOCK_ATTESTATION: &str = "unknown_block_attestation"; pub const UNKNOWN_BLOCK_AGGREGATE: &str = "unknown_block_aggregate"; pub const UNKNOWN_LIGHT_CLIENT_UPDATE: &str = "unknown_light_client_update"; pub const GOSSIP_BLS_TO_EXECUTION_CHANGE: &str = "gossip_bls_to_execution_change"; +pub const API_REQUEST_P0: &str = "api_request_p0"; +pub const API_REQUEST_P1: &str = "api_request_p1"; + +#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)] +pub struct BeaconProcessorConfig { + pub max_workers: usize, + pub max_work_event_queue_len: usize, + pub max_scheduled_work_queue_len: usize, + pub max_gossip_attestation_batch_size: usize, + pub max_gossip_aggregate_batch_size: usize, + pub enable_backfill_rate_limiting: bool, +} + +impl Default for BeaconProcessorConfig { + fn default() -> Self { + Self { + max_workers: cmp::max(1, num_cpus::get()), + max_work_event_queue_len: DEFAULT_MAX_WORK_EVENT_QUEUE_LEN, + max_scheduled_work_queue_len: DEFAULT_MAX_SCHEDULED_WORK_QUEUE_LEN, + max_gossip_attestation_batch_size: DEFAULT_MAX_GOSSIP_ATTESTATION_BATCH_SIZE, + max_gossip_aggregate_batch_size: DEFAULT_MAX_GOSSIP_AGGREGATE_BATCH_SIZE, + enable_backfill_rate_limiting: true, + } + } +} + +// The channels necessary to instantiate a `BeaconProcessor`. +pub struct BeaconProcessorChannels { + pub beacon_processor_tx: BeaconProcessorSend, + pub beacon_processor_rx: mpsc::Receiver>, + pub work_reprocessing_tx: mpsc::Sender, + pub work_reprocessing_rx: mpsc::Receiver, +} + +impl BeaconProcessorChannels { + pub fn new(config: &BeaconProcessorConfig) -> Self { + let (beacon_processor_tx, beacon_processor_rx) = + mpsc::channel(config.max_scheduled_work_queue_len); + let (work_reprocessing_tx, work_reprocessing_rx) = + mpsc::channel(config.max_scheduled_work_queue_len); + + Self { + beacon_processor_tx: BeaconProcessorSend(beacon_processor_tx), + beacon_processor_rx, + work_reprocessing_rx, + work_reprocessing_tx, + } + } +} + +impl Default for BeaconProcessorChannels { + fn default() -> Self { + Self::new(&BeaconProcessorConfig::default()) + } +} /// A simple first-in-first-out queue with a maximum length. struct FifoQueue { @@ -363,7 +427,7 @@ impl WorkEvent { } } -impl std::convert::From for WorkEvent { +impl From for WorkEvent { fn from(ready_work: ReadyWork) -> Self { match ready_work { ReadyWork::Block(QueuedGossipBlock { @@ -465,6 +529,10 @@ impl BeaconProcessorSend { pub type AsyncFn = Pin + Send + Sync>>; pub type BlockingFn = Box; pub type BlockingFnWithManualSendOnIdle = Box; +pub enum BlockingOrAsync { + Blocking(BlockingFn), + Async(AsyncFn), +} /// Indicates the type of work to be performed and therefore its priority and /// queuing specifics. @@ -523,6 +591,8 @@ pub enum Work { BlocksByRootsRequest(BlockingFnWithManualSendOnIdle), GossipBlsToExecutionChange(BlockingFn), LightClientBootstrapRequest(BlockingFn), + ApiRequestP0(BlockingOrAsync), + ApiRequestP1(BlockingOrAsync), } impl fmt::Debug for Work { @@ -560,6 +630,8 @@ impl Work { Work::UnknownBlockAggregate { .. } => UNKNOWN_BLOCK_AGGREGATE, Work::GossipBlsToExecutionChange(_) => GOSSIP_BLS_TO_EXECUTION_CHANGE, Work::UnknownLightClientOptimisticUpdate { .. } => UNKNOWN_LIGHT_CLIENT_UPDATE, + Work::ApiRequestP0 { .. } => API_REQUEST_P0, + Work::ApiRequestP1 { .. } => API_REQUEST_P1, } } } @@ -638,7 +710,7 @@ pub struct BeaconProcessor { pub executor: TaskExecutor, pub max_workers: usize, pub current_workers: usize, - pub enable_backfill_rate_limiting: bool, + pub config: BeaconProcessorConfig, pub log: Logger, } @@ -714,11 +786,13 @@ impl BeaconProcessor { let mut lcbootstrap_queue = FifoQueue::new(MAX_LIGHT_CLIENT_BOOTSTRAP_QUEUE_LEN); + let mut api_request_p0_queue = FifoQueue::new(MAX_API_REQUEST_P0_QUEUE_LEN); + let mut api_request_p1_queue = FifoQueue::new(MAX_API_REQUEST_P1_QUEUE_LEN); + // Channels for sending work to the re-process scheduler (`work_reprocessing_tx`) and to // receive them back once they are ready (`ready_work_rx`). let (ready_work_tx, ready_work_rx) = - mpsc::channel::(MAX_SCHEDULED_WORK_QUEUE_LEN); - + mpsc::channel::(self.config.max_scheduled_work_queue_len); spawn_reprocess_scheduler( ready_work_tx, work_reprocessing_rx, @@ -739,7 +813,7 @@ impl BeaconProcessor { reprocess_work_rx: ready_work_rx, }; - let enable_backfill_rate_limiting = self.enable_backfill_rate_limiting; + let enable_backfill_rate_limiting = self.config.enable_backfill_rate_limiting; loop { let work_event = match inbound_events.next().await { @@ -850,12 +924,17 @@ impl BeaconProcessor { // required to verify some attestations. } else if let Some(item) = gossip_block_queue.pop() { self.spawn_worker(item, idle_tx); + // Check the priority 0 API requests after blocks, but before attestations. + } else if let Some(item) = api_request_p0_queue.pop() { + self.spawn_worker(item, idle_tx); // Check the aggregates, *then* the unaggregates since we assume that // aggregates are more valuable to local validators and effectively give us // more information with less signature verification time. } else if aggregate_queue.len() > 0 { - let batch_size = - cmp::min(aggregate_queue.len(), MAX_GOSSIP_AGGREGATE_BATCH_SIZE); + let batch_size = cmp::min( + aggregate_queue.len(), + self.config.max_gossip_aggregate_batch_size, + ); if batch_size < 2 { // One single aggregate is in the queue, process it individually. @@ -914,7 +993,7 @@ impl BeaconProcessor { } else if attestation_queue.len() > 0 { let batch_size = cmp::min( attestation_queue.len(), - MAX_GOSSIP_ATTESTATION_BATCH_SIZE, + self.config.max_gossip_attestation_batch_size, ); if batch_size < 2 { @@ -1005,6 +1084,12 @@ impl BeaconProcessor { self.spawn_worker(item, idle_tx); } else if let Some(item) = gossip_bls_to_execution_change_queue.pop() { self.spawn_worker(item, idle_tx); + // Check the priority 1 API requests after we've + // processed all the interesting things from the network + // and things required for us to stay in good repute + // with our P2P peers. + } else if let Some(item) = api_request_p1_queue.pop() { + self.spawn_worker(item, idle_tx); // Handle backfill sync chain segments. } else if let Some(item) = backfill_chain_segment.pop() { self.spawn_worker(item, idle_tx); @@ -1127,6 +1212,12 @@ impl BeaconProcessor { Work::UnknownLightClientOptimisticUpdate { .. } => { unknown_light_client_update_queue.push(work, work_id, &self.log) } + Work::ApiRequestP0 { .. } => { + api_request_p0_queue.push(work, work_id, &self.log) + } + Work::ApiRequestP1 { .. } => { + api_request_p1_queue.push(work, work_id, &self.log) + } } } } @@ -1183,6 +1274,14 @@ impl BeaconProcessor { &metrics::BEACON_PROCESSOR_BLS_TO_EXECUTION_CHANGE_QUEUE_TOTAL, gossip_bls_to_execution_change_queue.len() as i64, ); + metrics::set_gauge( + &metrics::BEACON_PROCESSOR_API_REQUEST_P0_QUEUE_TOTAL, + api_request_p0_queue.len() as i64, + ); + metrics::set_gauge( + &metrics::BEACON_PROCESSOR_API_REQUEST_P1_QUEUE_TOTAL, + api_request_p1_queue.len() as i64, + ); if aggregate_queue.is_full() && aggregate_debounce.elapsed() { error!( @@ -1299,6 +1398,10 @@ impl BeaconProcessor { task_spawner.spawn_blocking_with_manual_send_idle(work) } Work::ChainSegmentBackfill(process_fn) => task_spawner.spawn_async(process_fn), + Work::ApiRequestP0(process_fn) | Work::ApiRequestP1(process_fn) => match process_fn { + BlockingOrAsync::Blocking(process_fn) => task_spawner.spawn_blocking(process_fn), + BlockingOrAsync::Async(process_fn) => task_spawner.spawn_async(process_fn), + }, Work::GossipVoluntaryExit(process_fn) | Work::GossipProposerSlashing(process_fn) | Work::GossipAttesterSlashing(process_fn) diff --git a/beacon_node/beacon_processor/src/metrics.rs b/beacon_node/beacon_processor/src/metrics.rs index 65ab0bd8fc5..e14c39e9a84 100644 --- a/beacon_node/beacon_processor/src/metrics.rs +++ b/beacon_node/beacon_processor/src/metrics.rs @@ -100,6 +100,15 @@ lazy_static::lazy_static! { "beacon_processor_sync_contribution_queue_total", "Count of sync committee contributions waiting to be processed." ); + // HTTP API requests. + pub static ref BEACON_PROCESSOR_API_REQUEST_P0_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_api_request_p0_queue_total", + "Count of P0 HTTP requesets waiting to be processed." + ); + pub static ref BEACON_PROCESSOR_API_REQUEST_P1_QUEUE_TOTAL: Result = try_create_int_gauge( + "beacon_processor_api_request_p1_queue_total", + "Count of P1 HTTP requesets waiting to be processed." + ); /* * Attestation reprocessing queue metrics. diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 71a9b28fb05..8383963b7c2 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -13,10 +13,8 @@ use beacon_chain::{ store::{HotColdDB, ItemStore, LevelDB, StoreConfig}, BeaconChain, BeaconChainTypes, Eth1ChainBackend, MigratorConfig, ServerSentEventHandler, }; -use beacon_processor::{ - work_reprocessing_queue::ReprocessQueueMessage, BeaconProcessor, BeaconProcessorSend, - WorkEvent, MAX_SCHEDULED_WORK_QUEUE_LEN, MAX_WORK_EVENT_QUEUE_LEN, -}; +use beacon_processor::BeaconProcessorConfig; +use beacon_processor::{BeaconProcessor, BeaconProcessorChannels}; use environment::RuntimeContext; use eth1::{Config as Eth1Config, Service as Eth1Service}; use eth2::{ @@ -37,7 +35,7 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use std::time::Duration; use timer::spawn_timer; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::oneshot; use types::{ test_utils::generate_deterministic_keypairs, BeaconState, ChainSpec, EthSpec, ExecutionBlockHash, Hash256, SignedBeaconBlock, @@ -76,11 +74,9 @@ pub struct ClientBuilder { http_api_config: http_api::Config, http_metrics_config: http_metrics::Config, slasher: Option>>, + beacon_processor_config: Option, + beacon_processor_channels: Option>, eth_spec_instance: T::EthSpec, - beacon_processor_send: BeaconProcessorSend, - beacon_processor_receive: mpsc::Receiver>, - work_reprocessing_tx: mpsc::Sender, - work_reprocessing_rx: mpsc::Receiver, } impl @@ -96,10 +92,6 @@ where /// /// The `eth_spec_instance` parameter is used to concretize `TEthSpec`. pub fn new(eth_spec_instance: TEthSpec) -> Self { - let (beacon_processor_send, beacon_processor_receive) = - mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN); - let (work_reprocessing_tx, work_reprocessing_rx) = - mpsc::channel(MAX_SCHEDULED_WORK_QUEUE_LEN); Self { slot_clock: None, store: None, @@ -117,10 +109,8 @@ where http_metrics_config: <_>::default(), slasher: None, eth_spec_instance, - beacon_processor_send: BeaconProcessorSend(beacon_processor_send), - beacon_processor_receive, - work_reprocessing_tx, - work_reprocessing_rx, + beacon_processor_config: None, + beacon_processor_channels: None, } } @@ -136,6 +126,12 @@ where self } + pub fn beacon_processor(mut self, config: BeaconProcessorConfig) -> Self { + self.beacon_processor_channels = Some(BeaconProcessorChannels::new(&config)); + self.beacon_processor_config = Some(config); + self + } + pub fn slasher(mut self, slasher: Arc>) -> Self { self.slasher = Some(slasher); self @@ -496,6 +492,7 @@ where chain: None, network_senders: None, network_globals: None, + beacon_processor_send: None, eth1_service: Some(genesis_service.eth1_service.clone()), log: context.log().clone(), sse_logging_components: runtime_context.sse_logging_components.clone(), @@ -573,6 +570,10 @@ where .as_ref() .ok_or("network requires a runtime_context")? .clone(); + let beacon_processor_channels = self + .beacon_processor_channels + .as_ref() + .ok_or("network requires beacon_processor_channels")?; // If gossipsub metrics are required we build a registry to record them let mut gossipsub_registry = if config.metrics_enabled { @@ -588,8 +589,8 @@ where gossipsub_registry .as_mut() .map(|registry| registry.sub_registry_with_prefix("gossipsub")), - self.beacon_processor_send.clone(), - self.work_reprocessing_tx.clone(), + beacon_processor_channels.beacon_processor_tx.clone(), + beacon_processor_channels.work_reprocessing_tx.clone(), ) .await .map_err(|e| format!("Failed to start network: {:?}", e))?; @@ -712,6 +713,14 @@ where .runtime_context .as_ref() .ok_or("build requires a runtime context")?; + let beacon_processor_channels = self + .beacon_processor_channels + .take() + .ok_or("build requires beacon_processor_channels")?; + let beacon_processor_config = self + .beacon_processor_config + .take() + .ok_or("build requires a beacon_processor_config")?; let log = runtime_context.log().clone(); let http_api_listen_addr = if self.http_api_config.enabled { @@ -721,6 +730,7 @@ where network_senders: self.network_senders.clone(), network_globals: self.network_globals.clone(), eth1_service: self.eth1_service.clone(), + beacon_processor_send: Some(beacon_processor_channels.beacon_processor_tx.clone()), sse_logging_components: runtime_context.sse_logging_components.clone(), log: log.clone(), }); @@ -784,15 +794,13 @@ where executor: beacon_processor_context.executor.clone(), max_workers: cmp::max(1, num_cpus::get()), current_workers: 0, - enable_backfill_rate_limiting: beacon_chain - .config - .enable_backfill_rate_limiting, + config: beacon_processor_config, log: beacon_processor_context.log().clone(), } .spawn_manager( - self.beacon_processor_receive, - self.work_reprocessing_tx, - self.work_reprocessing_rx, + beacon_processor_channels.beacon_processor_rx, + beacon_processor_channels.work_reprocessing_tx, + beacon_processor_channels.work_reprocessing_rx, None, beacon_chain.slot_clock.clone(), beacon_chain.spec.maximum_gossip_clock_disparity(), diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index 95a00b37492..b4deb52fc31 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -1,4 +1,5 @@ use beacon_chain::validator_monitor::DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD; +use beacon_processor::BeaconProcessorConfig; use directory::DEFAULT_ROOT_DIR; use environment::LoggerConfig; use network::NetworkConfig; @@ -80,6 +81,7 @@ pub struct Config { pub slasher: Option, pub logger_config: LoggerConfig, pub always_prefer_builder_payload: bool, + pub beacon_processor: BeaconProcessorConfig, } impl Default for Config { @@ -107,6 +109,7 @@ impl Default for Config { validator_monitor_individual_tracking_threshold: DEFAULT_INDIVIDUAL_TRACKING_THRESHOLD, logger_config: LoggerConfig::default(), always_prefer_builder_payload: false, + beacon_processor: <_>::default(), } } } diff --git a/beacon_node/http_api/Cargo.toml b/beacon_node/http_api/Cargo.toml index 4b4a28b51ec..0fa286f1655 100644 --- a/beacon_node/http_api/Cargo.toml +++ b/beacon_node/http_api/Cargo.toml @@ -3,12 +3,12 @@ name = "http_api" version = "0.1.0" authors = ["Paul Hauner "] edition = "2021" -autotests = false # using a single test binary compiles faster +autotests = false # using a single test binary compiles faster [dependencies] warp = { version = "0.3.2", features = ["tls"] } serde = { version = "1.0.116", features = ["derive"] } -tokio = { version = "1.14.0", features = ["macros","sync"] } +tokio = { version = "1.14.0", features = ["macros", "sync"] } tokio-stream = { version = "0.1.3", features = ["sync"] } types = { path = "../../consensus/types" } hex = "0.4.2" @@ -27,9 +27,9 @@ slot_clock = { path = "../../common/slot_clock" } ethereum_ssz = "0.5.0" bs58 = "0.4.0" futures = "0.3.8" -execution_layer = {path = "../execution_layer"} +execution_layer = { path = "../execution_layer" } parking_lot = "0.12.0" -safe_arith = {path = "../../consensus/safe_arith"} +safe_arith = { path = "../../consensus/safe_arith" } task_executor = { path = "../../common/task_executor" } lru = "0.7.7" tree_hash = "0.5.0" @@ -40,9 +40,10 @@ logging = { path = "../../common/logging" } ethereum_serde_utils = "0.5.0" operation_pool = { path = "../operation_pool" } sensitive_url = { path = "../../common/sensitive_url" } -unused_port = {path = "../../common/unused_port"} +unused_port = { path = "../../common/unused_port" } store = { path = "../store" } bytes = "1.1.0" +beacon_processor = { path = "../beacon_processor" } [dev-dependencies] environment = { path = "../../lighthouse/environment" } @@ -52,4 +53,4 @@ genesis = { path = "../genesis" } [[test]] name = "bn_http_api_tests" -path = "tests/main.rs" \ No newline at end of file +path = "tests/main.rs" diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 739371c6ee9..9512d18aba8 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -18,8 +18,10 @@ mod standard_block_rewards; mod state_id; mod sync_committee_rewards; mod sync_committees; +mod task_spawner; pub mod test_utils; mod ui; +mod validator; mod validator_inclusion; mod version; @@ -28,6 +30,7 @@ use beacon_chain::{ validator_monitor::timestamp_now, AttestationError as AttnError, BeaconChain, BeaconChainError, BeaconChainTypes, ProduceBlockVerification, WhenSlotSkipped, }; +use beacon_processor::BeaconProcessorSend; pub use block_id::BlockId; use bytes::Bytes; use directory::DEFAULT_ROOT_DIR; @@ -57,7 +60,11 @@ use std::pin::Pin; use std::sync::Arc; use sysinfo::{System, SystemExt}; use system_health::observe_system_health_bn; -use tokio::sync::mpsc::{Sender, UnboundedSender}; +use task_spawner::{Priority, TaskSpawner}; +use tokio::sync::{ + mpsc::{Sender, UnboundedSender}, + oneshot, +}; use tokio_stream::{wrappers::BroadcastStream, StreamExt}; use types::{ Attestation, AttestationData, AttestationShufflingId, AttesterSlashing, BeaconStateError, @@ -67,6 +74,7 @@ use types::{ SignedContributionAndProof, SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncCommitteeMessage, SyncContributionData, }; +use validator::pubkey_to_validator_index; use version::{ add_consensus_version_header, execution_optimistic_finalized_fork_versioned_response, fork_versioned_response, inconsistent_fork_rejection, unsupported_version_rejection, V1, V2, @@ -75,11 +83,7 @@ use warp::http::StatusCode; use warp::sse::Event; use warp::Reply; use warp::{http::Response, Filter}; -use warp_utils::{ - query::multi_key_query, - task::{blocking_json_task, blocking_response_task}, - uor::UnifyingOrFilter, -}; +use warp_utils::{query::multi_key_query, uor::UnifyingOrFilter}; const API_PREFIX: &str = "eth"; @@ -111,6 +115,7 @@ pub struct Context { pub chain: Option>>, pub network_senders: Option>, pub network_globals: Option>>, + pub beacon_processor_send: Option>, pub eth1_service: Option, pub sse_logging_components: Option, pub log: Logger, @@ -127,6 +132,7 @@ pub struct Config { pub allow_sync_stalled: bool, pub spec_fork_name: Option, pub data_dir: PathBuf, + pub enable_beacon_processor: bool, } impl Default for Config { @@ -140,6 +146,7 @@ impl Default for Config { allow_sync_stalled: false, spec_fork_name: None, data_dir: PathBuf::from(DEFAULT_ROOT_DIR), + enable_beacon_processor: true, } } } @@ -488,6 +495,14 @@ pub fn serve( let app_start = std::time::Instant::now(); let app_start_filter = warp::any().map(move || app_start); + // Create a `warp` filter that provides access to the `TaskSpawner`. + let beacon_processor_send = ctx + .beacon_processor_send + .clone() + .filter(|_| config.enable_beacon_processor); + let task_spawner_filter = + warp::any().map(move || TaskSpawner::new(beacon_processor_send.clone())); + /* * * Start of HTTP method definitions. @@ -499,17 +514,20 @@ pub fn serve( .and(warp::path("beacon")) .and(warp::path("genesis")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(|chain: Arc>| { - blocking_json_task(move || { - let genesis_data = api_types::GenesisData { - genesis_time: chain.genesis_time, - genesis_validators_root: chain.genesis_validators_root, - genesis_fork_version: chain.spec.genesis_fork_version, - }; - Ok(api_types::GenericResponse::from(genesis_data)) - }) - }); + .and_then( + |task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + let genesis_data = api_types::GenesisData { + genesis_time: chain.genesis_time, + genesis_validators_root: chain.genesis_validators_root, + genesis_fork_version: chain.spec.genesis_fork_version, + }; + Ok(api_types::GenericResponse::from(genesis_data)) + }) + }, + ); /* * beacon/states/{state_id} @@ -523,6 +541,7 @@ pub fn serve( "Invalid state ID".to_string(), )) })) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()); // GET beacon/states/{state_id}/root @@ -530,65 +549,77 @@ pub fn serve( .clone() .and(warp::path("root")) .and(warp::path::end()) - .and_then(|state_id: StateId, chain: Arc>| { - blocking_json_task(move || { - let (root, execution_optimistic, finalized) = state_id.root(&chain)?; - Ok(root) - .map(api_types::RootData::from) - .map(api_types::GenericResponse::from) - .map(|resp| { - resp.add_execution_optimistic_finalized(execution_optimistic, finalized) - }) - }) - }); + .and_then( + |state_id: StateId, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + let (root, execution_optimistic, finalized) = state_id.root(&chain)?; + Ok(root) + .map(api_types::RootData::from) + .map(api_types::GenericResponse::from) + .map(|resp| { + resp.add_execution_optimistic_finalized(execution_optimistic, finalized) + }) + }) + }, + ); // GET beacon/states/{state_id}/fork let get_beacon_state_fork = beacon_states_path .clone() .and(warp::path("fork")) .and(warp::path::end()) - .and_then(|state_id: StateId, chain: Arc>| { - blocking_json_task(move || { - let (fork, execution_optimistic, finalized) = - state_id.fork_and_execution_optimistic_and_finalized(&chain)?; - Ok(api_types::ExecutionOptimisticFinalizedResponse { - data: fork, - execution_optimistic: Some(execution_optimistic), - finalized: Some(finalized), + .and_then( + |state_id: StateId, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + let (fork, execution_optimistic, finalized) = + state_id.fork_and_execution_optimistic_and_finalized(&chain)?; + Ok(api_types::ExecutionOptimisticFinalizedResponse { + data: fork, + execution_optimistic: Some(execution_optimistic), + finalized: Some(finalized), + }) }) - }) - }); + }, + ); // GET beacon/states/{state_id}/finality_checkpoints let get_beacon_state_finality_checkpoints = beacon_states_path .clone() .and(warp::path("finality_checkpoints")) .and(warp::path::end()) - .and_then(|state_id: StateId, chain: Arc>| { - blocking_json_task(move || { - let (data, execution_optimistic, finalized) = state_id - .map_state_and_execution_optimistic_and_finalized( - &chain, - |state, execution_optimistic, finalized| { - Ok(( - api_types::FinalityCheckpointsData { - previous_justified: state.previous_justified_checkpoint(), - current_justified: state.current_justified_checkpoint(), - finalized: state.finalized_checkpoint(), - }, - execution_optimistic, - finalized, - )) - }, - )?; + .and_then( + |state_id: StateId, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + let (data, execution_optimistic, finalized) = state_id + .map_state_and_execution_optimistic_and_finalized( + &chain, + |state, execution_optimistic, finalized| { + Ok(( + api_types::FinalityCheckpointsData { + previous_justified: state.previous_justified_checkpoint(), + current_justified: state.current_justified_checkpoint(), + finalized: state.finalized_checkpoint(), + }, + execution_optimistic, + finalized, + )) + }, + )?; - Ok(api_types::ExecutionOptimisticFinalizedResponse { - data, - execution_optimistic: Some(execution_optimistic), - finalized: Some(finalized), + Ok(api_types::ExecutionOptimisticFinalizedResponse { + data, + execution_optimistic: Some(execution_optimistic), + finalized: Some(finalized), + }) }) - }) - }); + }, + ); // GET beacon/states/{state_id}/validator_balances?id let get_beacon_state_validator_balances = beacon_states_path @@ -598,9 +629,10 @@ pub fn serve( .and(multi_key_query::()) .and_then( |state_id: StateId, + task_spawner: TaskSpawner, chain: Arc>, query_res: Result| { - blocking_json_task(move || { + task_spawner.blocking_json_task(Priority::P1, move || { let query = query_res?; let (data, execution_optimistic, finalized) = state_id .map_state_and_execution_optimistic_and_finalized( @@ -655,9 +687,10 @@ pub fn serve( .and(multi_key_query::()) .and_then( |state_id: StateId, + task_spawner: TaskSpawner, chain: Arc>, query_res: Result| { - blocking_json_task(move || { + task_spawner.blocking_json_task(Priority::P1, move || { let query = query_res?; let (data, execution_optimistic, finalized) = state_id .map_state_and_execution_optimistic_and_finalized( @@ -737,16 +770,24 @@ pub fn serve( })) .and(warp::path::end()) .and_then( - |state_id: StateId, chain: Arc>, validator_id: ValidatorId| { - blocking_json_task(move || { + |state_id: StateId, + task_spawner: TaskSpawner, + chain: Arc>, + validator_id: ValidatorId| { + task_spawner.blocking_json_task(Priority::P1, move || { let (data, execution_optimistic, finalized) = state_id .map_state_and_execution_optimistic_and_finalized( &chain, |state, execution_optimistic, finalized| { let index_opt = match &validator_id { - ValidatorId::PublicKey(pubkey) => { - state.validators().iter().position(|v| v.pubkey == *pubkey) - } + ValidatorId::PublicKey(pubkey) => pubkey_to_validator_index( + &chain, state, pubkey, + ) + .map_err(|e| { + warp_utils::reject::custom_not_found(format!( + "unable to access pubkey cache: {e:?}", + )) + })?, ValidatorId::Index(index) => Some(*index as usize), }; @@ -797,8 +838,11 @@ pub fn serve( .and(warp::query::()) .and(warp::path::end()) .and_then( - |state_id: StateId, chain: Arc>, query: api_types::CommitteesQuery| { - blocking_json_task(move || { + |state_id: StateId, + task_spawner: TaskSpawner, + chain: Arc>, + query: api_types::CommitteesQuery| { + task_spawner.blocking_json_task(Priority::P1, move || { let (data, execution_optimistic, finalized) = state_id .map_state_and_execution_optimistic_and_finalized( &chain, @@ -978,9 +1022,10 @@ pub fn serve( .and(warp::path::end()) .and_then( |state_id: StateId, + task_spawner: TaskSpawner, chain: Arc>, query: api_types::SyncCommitteesQuery| { - blocking_json_task(move || { + task_spawner.blocking_json_task(Priority::P1, move || { let (sync_committee, execution_optimistic, finalized) = state_id .map_state_and_execution_optimistic_and_finalized( &chain, @@ -1042,8 +1087,11 @@ pub fn serve( .and(warp::query::()) .and(warp::path::end()) .and_then( - |state_id: StateId, chain: Arc>, query: api_types::RandaoQuery| { - blocking_json_task(move || { + |state_id: StateId, + task_spawner: TaskSpawner, + chain: Arc>, + query: api_types::RandaoQuery| { + task_spawner.blocking_json_task(Priority::P1, move || { let (randao, execution_optimistic, finalized) = state_id .map_state_and_execution_optimistic_and_finalized( &chain, @@ -1078,10 +1126,13 @@ pub fn serve( .and(warp::path("headers")) .and(warp::query::()) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and_then( - |query: api_types::HeadersQuery, chain: Arc>| { - blocking_json_task(move || { + |query: api_types::HeadersQuery, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { let (root, block, execution_optimistic, finalized) = match (query.slot, query.parent_root) { // No query parameters, return the canonical head block. @@ -1175,36 +1226,41 @@ pub fn serve( )) })) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(|block_id: BlockId, chain: Arc>| { - blocking_json_task(move || { - let (root, execution_optimistic, finalized) = block_id.root(&chain)?; - // Ignore the second `execution_optimistic` since the first one has more - // information about the original request. - let (block, _execution_optimistic, _finalized) = - BlockId::from_root(root).blinded_block(&chain)?; - - let canonical = chain - .block_root_at_slot(block.slot(), WhenSlotSkipped::None) - .map_err(warp_utils::reject::beacon_chain_error)? - .map_or(false, |canonical| root == canonical); - - let data = api_types::BlockHeaderData { - root, - canonical, - header: api_types::BlockHeaderAndSignature { - message: block.message().block_header(), - signature: block.signature().clone().into(), - }, - }; + .and_then( + |block_id: BlockId, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + let (root, execution_optimistic, finalized) = block_id.root(&chain)?; + // Ignore the second `execution_optimistic` since the first one has more + // information about the original request. + let (block, _execution_optimistic, _finalized) = + BlockId::from_root(root).blinded_block(&chain)?; + + let canonical = chain + .block_root_at_slot(block.slot(), WhenSlotSkipped::None) + .map_err(warp_utils::reject::beacon_chain_error)? + .map_or(false, |canonical| root == canonical); + + let data = api_types::BlockHeaderData { + root, + canonical, + header: api_types::BlockHeaderAndSignature { + message: block.message().block_header(), + signature: block.signature().clone().into(), + }, + }; - Ok(api_types::ExecutionOptimisticFinalizedResponse { - execution_optimistic: Some(execution_optimistic), - finalized: Some(finalized), - data, + Ok(api_types::ExecutionOptimisticFinalizedResponse { + execution_optimistic: Some(execution_optimistic), + finalized: Some(finalized), + data, + }) }) - }) - }); + }, + ); /* * beacon/blocks @@ -1216,24 +1272,28 @@ pub fn serve( .and(warp::path("blocks")) .and(warp::path::end()) .and(warp::body::json()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(network_tx_filter.clone()) .and(log_filter.clone()) .and_then( |block: Arc>, + task_spawner: TaskSpawner, chain: Arc>, network_tx: UnboundedSender>, - log: Logger| async move { - publish_blocks::publish_block( - None, - ProvenancedBlock::local(block), - chain, - &network_tx, - log, - BroadcastValidation::default(), - ) - .await - .map(|()| warp::reply().into_response()) + log: Logger| { + task_spawner.spawn_async_with_rejection(Priority::P0, async move { + publish_blocks::publish_block( + None, + ProvenancedBlock::local(block), + chain, + &network_tx, + log, + BroadcastValidation::default(), + ) + .await + .map(|()| warp::reply().into_response()) + }) }, ); @@ -1278,35 +1338,39 @@ pub fn serve( .and(warp::query::()) .and(warp::path::end()) .and(warp::body::json()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( |validation_level: api_types::BroadcastValidationQuery, block: Arc>, + task_spawner: TaskSpawner, chain: Arc>, network_tx: UnboundedSender>, - log: Logger| async move { - match publish_blocks::publish_block( - None, - ProvenancedBlock::local(block), - chain, - &network_tx, - log, - validation_level.broadcast_validation, - ) - .await - { - Ok(()) => warp::reply().into_response(), - Err(e) => match warp_utils::reject::handle_rejection(e).await { - Ok(reply) => reply.into_response(), - Err(_) => warp::reply::with_status( - StatusCode::INTERNAL_SERVER_ERROR, - eth2::StatusCode::INTERNAL_SERVER_ERROR, - ) - .into_response(), - }, - } + log: Logger| { + task_spawner.spawn_async(Priority::P1, async move { + match publish_blocks::publish_block( + None, + ProvenancedBlock::local(block), + chain, + &network_tx, + log, + validation_level.broadcast_validation, + ) + .await + { + Ok(()) => warp::reply().into_response(), + Err(e) => match warp_utils::reject::handle_rejection(e).await { + Ok(reply) => reply.into_response(), + Err(_) => warp::reply::with_status( + StatusCode::INTERNAL_SERVER_ERROR, + eth2::StatusCode::INTERNAL_SERVER_ERROR, + ) + .into_response(), + }, + } + }) }, ); @@ -1371,23 +1435,27 @@ pub fn serve( .and(warp::path("blinded_blocks")) .and(warp::path::end()) .and(warp::body::json()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(network_tx_filter.clone()) .and(log_filter.clone()) .and_then( |block: SignedBeaconBlock>, + task_spawner: TaskSpawner, chain: Arc>, network_tx: UnboundedSender>, - log: Logger| async move { - publish_blocks::publish_blinded_block( - block, - chain, - &network_tx, - log, - BroadcastValidation::default(), - ) - .await - .map(|()| warp::reply().into_response()) + log: Logger| { + task_spawner.spawn_async_with_rejection(Priority::P0, async move { + publish_blocks::publish_blinded_block( + block, + chain, + &network_tx, + log, + BroadcastValidation::default(), + ) + .await + .map(|()| warp::reply().into_response()) + }) }, ); @@ -1437,34 +1505,38 @@ pub fn serve( .and(warp::query::()) .and(warp::path::end()) .and(warp::body::json()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(network_tx_filter.clone()) .and(log_filter.clone()) .then( |validation_level: api_types::BroadcastValidationQuery, block: SignedBeaconBlock>, + task_spawner: TaskSpawner, chain: Arc>, network_tx: UnboundedSender>, - log: Logger| async move { - match publish_blocks::publish_blinded_block( - block, - chain, - &network_tx, - log, - validation_level.broadcast_validation, - ) - .await - { - Ok(()) => warp::reply().into_response(), - Err(e) => match warp_utils::reject::handle_rejection(e).await { - Ok(reply) => reply.into_response(), - Err(_) => warp::reply::with_status( - StatusCode::INTERNAL_SERVER_ERROR, - eth2::StatusCode::INTERNAL_SERVER_ERROR, - ) - .into_response(), - }, - } + log: Logger| { + task_spawner.spawn_async(Priority::P0, async move { + match publish_blocks::publish_blinded_block( + block, + chain, + &network_tx, + log, + validation_level.broadcast_validation, + ) + .await + { + Ok(()) => warp::reply().into_response(), + Err(e) => match warp_utils::reject::handle_rejection(e).await { + Ok(reply) => reply.into_response(), + Err(_) => warp::reply::with_status( + StatusCode::INTERNAL_SERVER_ERROR, + eth2::StatusCode::INTERNAL_SERVER_ERROR, + ) + .into_response(), + }, + } + }) }, ); @@ -1530,12 +1602,14 @@ pub fn serve( .and(warp::path("beacon")) .and(warp::path("blocks")) .and(block_id_or_err) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()); let beacon_blocks_path_any = any_version .and(warp::path("beacon")) .and(warp::path("blocks")) .and(block_id_or_err) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()); // GET beacon/blocks/{block_id} @@ -1546,9 +1620,10 @@ pub fn serve( .and_then( |endpoint_version: EndpointVersion, block_id: BlockId, + task_spawner: TaskSpawner, chain: Arc>, accept_header: Option| { - async move { + task_spawner.spawn_async_with_rejection(Priority::P1, async move { let (block, execution_optimistic, finalized) = block_id.full_block(&chain).await?; let fork_name = block @@ -1576,7 +1651,7 @@ pub fn serve( .map(|res| warp::reply::json(&res).into_response()), } .map(|resp| add_consensus_version_header(resp, fork_name)) - } + }) }, ); @@ -1585,44 +1660,56 @@ pub fn serve( .clone() .and(warp::path("root")) .and(warp::path::end()) - .and_then(|block_id: BlockId, chain: Arc>| { - blocking_json_task(move || { - let (block, execution_optimistic, finalized) = block_id.blinded_block(&chain)?; - Ok(api_types::GenericResponse::from(api_types::RootData::from( - block.canonical_root(), - )) - .add_execution_optimistic_finalized(execution_optimistic, finalized)) - }) - }); + .and_then( + |block_id: BlockId, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + let (block, execution_optimistic, finalized) = + block_id.blinded_block(&chain)?; + Ok(api_types::GenericResponse::from(api_types::RootData::from( + block.canonical_root(), + )) + .add_execution_optimistic_finalized(execution_optimistic, finalized)) + }) + }, + ); // GET beacon/blocks/{block_id}/attestations let get_beacon_block_attestations = beacon_blocks_path_v1 .clone() .and(warp::path("attestations")) .and(warp::path::end()) - .and_then(|block_id: BlockId, chain: Arc>| { - blocking_json_task(move || { - let (block, execution_optimistic, finalized) = block_id.blinded_block(&chain)?; - Ok( - api_types::GenericResponse::from(block.message().body().attestations().clone()) - .add_execution_optimistic_finalized(execution_optimistic, finalized), - ) - }) - }); + .and_then( + |block_id: BlockId, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + let (block, execution_optimistic, finalized) = + block_id.blinded_block(&chain)?; + Ok(api_types::GenericResponse::from( + block.message().body().attestations().clone(), + ) + .add_execution_optimistic_finalized(execution_optimistic, finalized)) + }) + }, + ); // GET beacon/blinded_blocks/{block_id} let get_beacon_blinded_block = eth_v1 .and(warp::path("beacon")) .and(warp::path("blinded_blocks")) .and(block_id_or_err) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(warp::path::end()) .and(warp::header::optional::("accept")) .and_then( |block_id: BlockId, + task_spawner: TaskSpawner, chain: Arc>, accept_header: Option| { - blocking_response_task(move || { + task_spawner.blocking_response_task(Priority::P1, move || { let (block, execution_optimistic, finalized) = block_id.blinded_block(&chain)?; let fork_name = block @@ -1664,6 +1751,7 @@ pub fn serve( let beacon_pool_path = eth_v1 .and(warp::path("beacon")) .and(warp::path("pool")) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()); // POST beacon/pool/attestations @@ -1675,11 +1763,12 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .and_then( - |chain: Arc>, + |task_spawner: TaskSpawner, + chain: Arc>, attestations: Vec>, network_tx: UnboundedSender>, log: Logger| { - blocking_json_task(move || { + task_spawner.blocking_json_task(Priority::P0, move || { let seen_timestamp = timestamp_now(); let mut failures = Vec::new(); let mut num_already_known = 0; @@ -1816,8 +1905,10 @@ pub fn serve( .and(warp::path::end()) .and(warp::query::()) .and_then( - |chain: Arc>, query: api_types::AttestationPoolQuery| { - blocking_json_task(move || { + |task_spawner: TaskSpawner, + chain: Arc>, + query: api_types::AttestationPoolQuery| { + task_spawner.blocking_json_task(Priority::P1, move || { let query_filter = |data: &AttestationData| { query.slot.map_or(true, |slot| slot == data.slot) && query @@ -1847,10 +1938,11 @@ pub fn serve( .and(warp::body::json()) .and(network_tx_filter.clone()) .and_then( - |chain: Arc>, + |task_spawner: TaskSpawner, + chain: Arc>, slashing: AttesterSlashing, network_tx: UnboundedSender>| { - blocking_json_task(move || { + task_spawner.blocking_json_task(Priority::P0, move || { let outcome = chain .verify_attester_slashing_for_gossip(slashing.clone()) .map_err(|e| { @@ -1887,12 +1979,14 @@ pub fn serve( .clone() .and(warp::path("attester_slashings")) .and(warp::path::end()) - .and_then(|chain: Arc>| { - blocking_json_task(move || { - let attestations = chain.op_pool.get_all_attester_slashings(); - Ok(api_types::GenericResponse::from(attestations)) - }) - }); + .and_then( + |task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + let attestations = chain.op_pool.get_all_attester_slashings(); + Ok(api_types::GenericResponse::from(attestations)) + }) + }, + ); // POST beacon/pool/proposer_slashings let post_beacon_pool_proposer_slashings = beacon_pool_path @@ -1902,10 +1996,11 @@ pub fn serve( .and(warp::body::json()) .and(network_tx_filter.clone()) .and_then( - |chain: Arc>, + |task_spawner: TaskSpawner, + chain: Arc>, slashing: ProposerSlashing, network_tx: UnboundedSender>| { - blocking_json_task(move || { + task_spawner.blocking_json_task(Priority::P0, move || { let outcome = chain .verify_proposer_slashing_for_gossip(slashing.clone()) .map_err(|e| { @@ -1942,13 +2037,15 @@ pub fn serve( .clone() .and(warp::path("proposer_slashings")) .and(warp::path::end()) - .and_then(|chain: Arc>| { - blocking_json_task(move || { - let attestations = chain.op_pool.get_all_proposer_slashings(); - Ok(api_types::GenericResponse::from(attestations)) - }) - }); - + .and_then( + |task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + let attestations = chain.op_pool.get_all_proposer_slashings(); + Ok(api_types::GenericResponse::from(attestations)) + }) + }, + ); + // POST beacon/pool/voluntary_exits let post_beacon_pool_voluntary_exits = beacon_pool_path .clone() @@ -1957,10 +2054,11 @@ pub fn serve( .and(warp::body::json()) .and(network_tx_filter.clone()) .and_then( - |chain: Arc>, + |task_spawner: TaskSpawner, + chain: Arc>, exit: SignedVoluntaryExit, network_tx: UnboundedSender>| { - blocking_json_task(move || { + task_spawner.blocking_json_task(Priority::P0, move || { let outcome = chain .verify_voluntary_exit_for_gossip(exit.clone()) .map_err(|e| { @@ -1995,12 +2093,14 @@ pub fn serve( .clone() .and(warp::path("voluntary_exits")) .and(warp::path::end()) - .and_then(|chain: Arc>| { - blocking_json_task(move || { - let attestations = chain.op_pool.get_all_voluntary_exits(); - Ok(api_types::GenericResponse::from(attestations)) - }) - }); + .and_then( + |task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + let attestations = chain.op_pool.get_all_voluntary_exits(); + Ok(api_types::GenericResponse::from(attestations)) + }) + }, + ); // POST beacon/pool/sync_committees let post_beacon_pool_sync_committees = beacon_pool_path @@ -2011,11 +2111,12 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .and_then( - |chain: Arc>, + |task_spawner: TaskSpawner, + chain: Arc>, signatures: Vec, network_tx: UnboundedSender>, log: Logger| { - blocking_json_task(move || { + task_spawner.blocking_json_task(Priority::P0, move || { sync_committees::process_sync_committee_signatures( signatures, network_tx, &chain, log, )?; @@ -2029,12 +2130,14 @@ pub fn serve( .clone() .and(warp::path("bls_to_execution_changes")) .and(warp::path::end()) - .and_then(|chain: Arc>| { - blocking_json_task(move || { - let address_changes = chain.op_pool.get_all_bls_to_execution_changes(); - Ok(api_types::GenericResponse::from(address_changes)) - }) - }); + .and_then( + |task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + let address_changes = chain.op_pool.get_all_bls_to_execution_changes(); + Ok(api_types::GenericResponse::from(address_changes)) + }) + }, + ); // POST beacon/pool/bls_to_execution_changes let post_beacon_pool_bls_to_execution_changes = beacon_pool_path @@ -2045,11 +2148,12 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .and_then( - |chain: Arc>, + |task_spawner: TaskSpawner, + chain: Arc>, address_changes: Vec, network_tx: UnboundedSender>, log: Logger| { - blocking_json_task(move || { + task_spawner.blocking_json_task(Priority::P0, move || { let mut failures = vec![]; for (index, address_change) in address_changes.into_iter().enumerate() { @@ -2133,10 +2237,13 @@ pub fn serve( .and(warp::path("deposit_snapshot")) .and(warp::path::end()) .and(warp::header::optional::("accept")) + .and(task_spawner_filter.clone()) .and(eth1_service_filter.clone()) .and_then( - |accept_header: Option, eth1_service: eth1::Service| { - blocking_response_task(move || match accept_header { + |accept_header: Option, + task_spawner: TaskSpawner, + eth1_service: eth1::Service| { + task_spawner.blocking_response_task(Priority::P1, move || match accept_header { Some(api_types::Accept::Json) | None => { let snapshot = eth1_service.get_deposit_snapshot(); Ok( @@ -2177,6 +2284,7 @@ pub fn serve( let beacon_rewards_path = eth_v1 .and(warp::path("beacon")) .and(warp::path("rewards")) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()); // GET beacon/rewards/blocks/{block_id} @@ -2185,17 +2293,21 @@ pub fn serve( .and(warp::path("blocks")) .and(block_id_or_err) .and(warp::path::end()) - .and_then(|chain: Arc>, block_id: BlockId| { - blocking_json_task(move || { - let (rewards, execution_optimistic, finalized) = - standard_block_rewards::compute_beacon_block_rewards(chain, block_id)?; - Ok(rewards) - .map(api_types::GenericResponse::from) - .map(|resp| { - resp.add_execution_optimistic_finalized(execution_optimistic, finalized) - }) - }) - }); + .and_then( + |task_spawner: TaskSpawner, + chain: Arc>, + block_id: BlockId| { + task_spawner.blocking_json_task(Priority::P1, move || { + let (rewards, execution_optimistic, finalized) = + standard_block_rewards::compute_beacon_block_rewards(chain, block_id)?; + Ok(rewards) + .map(api_types::GenericResponse::from) + .map(|resp| { + resp.add_execution_optimistic_finalized(execution_optimistic, finalized) + }) + }) + }, + ); /* * beacon/rewards @@ -2204,6 +2316,7 @@ pub fn serve( let beacon_rewards_path = eth_v1 .and(warp::path("beacon")) .and(warp::path("rewards")) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()); // POST beacon/rewards/attestations/{epoch} @@ -2214,8 +2327,11 @@ pub fn serve( .and(warp::path::end()) .and(warp::body::json()) .and_then( - |chain: Arc>, epoch: Epoch, validators: Vec| { - blocking_json_task(move || { + |task_spawner: TaskSpawner, + chain: Arc>, + epoch: Epoch, + validators: Vec| { + task_spawner.blocking_json_task(Priority::P1, move || { let attestation_rewards = chain .compute_attestation_rewards(epoch, validators) .map_err(|e| match e { @@ -2263,11 +2379,12 @@ pub fn serve( .and(warp::body::json()) .and(log_filter.clone()) .and_then( - |chain: Arc>, + |task_spawner: TaskSpawner, + chain: Arc>, block_id: BlockId, validators: Vec, log: Logger| { - blocking_json_task(move || { + task_spawner.blocking_json_task(Priority::P1, move || { let (rewards, execution_optimistic, finalized) = sync_committee_rewards::compute_sync_committee_rewards( chain, block_id, validators, log, @@ -2292,46 +2409,55 @@ pub fn serve( let get_config_fork_schedule = config_path .and(warp::path("fork_schedule")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(|chain: Arc>| { - blocking_json_task(move || { - let forks = ForkName::list_all() - .into_iter() - .filter_map(|fork_name| chain.spec.fork_for_name(fork_name)) - .collect::>(); - Ok(api_types::GenericResponse::from(forks)) - }) - }); + .and_then( + |task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + let forks = ForkName::list_all() + .into_iter() + .filter_map(|fork_name| chain.spec.fork_for_name(fork_name)) + .collect::>(); + Ok(api_types::GenericResponse::from(forks)) + }) + }, + ); // GET config/spec let spec_fork_name = ctx.config.spec_fork_name; let get_config_spec = config_path .and(warp::path("spec")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(move |chain: Arc>| { - blocking_json_task(move || { - let config_and_preset = - ConfigAndPreset::from_chain_spec::(&chain.spec, spec_fork_name); - Ok(api_types::GenericResponse::from(config_and_preset)) - }) - }); + .and_then( + move |task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P0, move || { + let config_and_preset = + ConfigAndPreset::from_chain_spec::(&chain.spec, spec_fork_name); + Ok(api_types::GenericResponse::from(config_and_preset)) + }) + }, + ); // GET config/deposit_contract let get_config_deposit_contract = config_path .and(warp::path("deposit_contract")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(|chain: Arc>| { - blocking_json_task(move || { - Ok(api_types::GenericResponse::from( - api_types::DepositContractData { - address: chain.spec.deposit_contract_address, - chain_id: chain.spec.deposit_chain_id, - }, - )) - }) - }); + .and_then( + |task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + Ok(api_types::GenericResponse::from( + api_types::DepositContractData { + address: chain.spec.deposit_contract_address, + chain_id: chain.spec.deposit_chain_id, + }, + )) + }) + }, + ); /* * debug @@ -2349,13 +2475,15 @@ pub fn serve( })) .and(warp::path::end()) .and(warp::header::optional::("accept")) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and_then( |endpoint_version: EndpointVersion, state_id: StateId, accept_header: Option, + task_spawner: TaskSpawner, chain: Arc>| { - blocking_response_task(move || match accept_header { + task_spawner.blocking_response_task(Priority::P1, move || match accept_header { Some(api_types::Accept::Ssz) => { // We can ignore the optimistic status for the "fork" since it's a // specification constant that doesn't change across competing heads of the @@ -2407,10 +2535,13 @@ pub fn serve( .and(warp::path("beacon")) .and(warp::path("heads")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and_then( - |endpoint_version: EndpointVersion, chain: Arc>| { - blocking_json_task(move || { + |endpoint_version: EndpointVersion, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { let heads = chain .heads() .into_iter() @@ -2443,48 +2574,51 @@ pub fn serve( .and(warp::path("debug")) .and(warp::path("fork_choice")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(|chain: Arc>| { - blocking_json_task(move || { - let beacon_fork_choice = chain.canonical_head.fork_choice_read_lock(); - - let proto_array = beacon_fork_choice.proto_array().core_proto_array(); - - let fork_choice_nodes = proto_array - .nodes - .iter() - .map(|node| { - let execution_status = if node.execution_status.is_execution_enabled() { - Some(node.execution_status.to_string()) - } else { - None - }; + .and_then( + |task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + let beacon_fork_choice = chain.canonical_head.fork_choice_read_lock(); - ForkChoiceNode { - slot: node.slot, - block_root: node.root, - parent_root: node - .parent - .and_then(|index| proto_array.nodes.get(index)) - .map(|parent| parent.root), - justified_epoch: node.justified_checkpoint.epoch, - finalized_epoch: node.finalized_checkpoint.epoch, - weight: node.weight, - validity: execution_status, - execution_block_hash: node - .execution_status - .block_hash() - .map(|block_hash| block_hash.into_root()), - } + let proto_array = beacon_fork_choice.proto_array().core_proto_array(); + + let fork_choice_nodes = proto_array + .nodes + .iter() + .map(|node| { + let execution_status = if node.execution_status.is_execution_enabled() { + Some(node.execution_status.to_string()) + } else { + None + }; + + ForkChoiceNode { + slot: node.slot, + block_root: node.root, + parent_root: node + .parent + .and_then(|index| proto_array.nodes.get(index)) + .map(|parent| parent.root), + justified_epoch: node.justified_checkpoint.epoch, + finalized_epoch: node.finalized_checkpoint.epoch, + weight: node.weight, + validity: execution_status, + execution_block_hash: node + .execution_status + .block_hash() + .map(|block_hash| block_hash.into_root()), + } + }) + .collect::>(); + Ok(ForkChoice { + justified_checkpoint: proto_array.justified_checkpoint, + finalized_checkpoint: proto_array.finalized_checkpoint, + fork_choice_nodes, }) - .collect::>(); - Ok(ForkChoice { - justified_checkpoint: proto_array.justified_checkpoint, - finalized_checkpoint: proto_array.finalized_checkpoint, - fork_choice_nodes, }) - }) - }); + }, + ); /* * node @@ -2495,50 +2629,54 @@ pub fn serve( .and(warp::path("node")) .and(warp::path("identity")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(network_globals.clone()) - .and_then(|network_globals: Arc>| { - blocking_json_task(move || { - let enr = network_globals.local_enr(); - let p2p_addresses = enr.multiaddr_p2p_tcp(); - let discovery_addresses = enr.multiaddr_p2p_udp(); - let meta_data = network_globals.local_metadata.read(); - Ok(api_types::GenericResponse::from(api_types::IdentityData { - peer_id: network_globals.local_peer_id().to_base58(), - enr, - p2p_addresses, - discovery_addresses, - metadata: api_types::MetaData { - seq_number: *meta_data.seq_number(), - attnets: format!( - "0x{}", - hex::encode(meta_data.attnets().clone().into_bytes()), - ), - syncnets: format!( - "0x{}", - hex::encode( - meta_data - .syncnets() - .map(|x| x.clone()) - .unwrap_or_default() - .into_bytes() - ) - ), - }, - })) - }) - }); + .and_then( + |task_spawner: TaskSpawner, + network_globals: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + let enr = network_globals.local_enr(); + let p2p_addresses = enr.multiaddr_p2p_tcp(); + let discovery_addresses = enr.multiaddr_p2p_udp(); + let meta_data = network_globals.local_metadata.read(); + Ok(api_types::GenericResponse::from(api_types::IdentityData { + peer_id: network_globals.local_peer_id().to_base58(), + enr, + p2p_addresses, + discovery_addresses, + metadata: api_types::MetaData { + seq_number: *meta_data.seq_number(), + attnets: format!( + "0x{}", + hex::encode(meta_data.attnets().clone().into_bytes()), + ), + syncnets: format!( + "0x{}", + hex::encode( + meta_data + .syncnets() + .map(|x| x.clone()) + .unwrap_or_default() + .into_bytes() + ) + ), + }, + })) + }) + }, + ); // GET node/version let get_node_version = eth_v1 .and(warp::path("node")) .and(warp::path("version")) .and(warp::path::end()) - .and_then(|| { - blocking_json_task(move || { - Ok(api_types::GenericResponse::from(api_types::VersionData { - version: version_with_platform(), - })) - }) + // Bypass the `task_spawner` since this method returns a static string. + .then(|| async { + warp::reply::json(&api_types::GenericResponse::from(api_types::VersionData { + version: version_with_platform(), + })) + .into_response() }); // GET node/syncing @@ -2546,10 +2684,13 @@ pub fn serve( .and(warp::path("node")) .and(warp::path("syncing")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(network_globals.clone()) .and(chain_filter.clone()) .and_then( - |network_globals: Arc>, chain: Arc>| { + |task_spawner: TaskSpawner, + network_globals: Arc>, + chain: Arc>| { async move { let el_offline = if let Some(el) = &chain.execution_layer { el.is_offline_or_erroring().await @@ -2557,32 +2698,34 @@ pub fn serve( true }; - blocking_json_task(move || { - let head_slot = chain.canonical_head.cached_head().head_slot(); - let current_slot = chain.slot_clock.now_or_genesis().ok_or_else(|| { - warp_utils::reject::custom_server_error( - "Unable to read slot clock".into(), - ) - })?; + task_spawner + .blocking_json_task(Priority::P0, move || { + let head_slot = chain.canonical_head.cached_head().head_slot(); + let current_slot = + chain.slot_clock.now_or_genesis().ok_or_else(|| { + warp_utils::reject::custom_server_error( + "Unable to read slot clock".into(), + ) + })?; - // Taking advantage of saturating subtraction on slot. - let sync_distance = current_slot - head_slot; + // Taking advantage of saturating subtraction on slot. + let sync_distance = current_slot - head_slot; - let is_optimistic = chain - .is_optimistic_or_invalid_head() - .map_err(warp_utils::reject::beacon_chain_error)?; + let is_optimistic = chain + .is_optimistic_or_invalid_head() + .map_err(warp_utils::reject::beacon_chain_error)?; - let syncing_data = api_types::SyncingData { - is_syncing: network_globals.sync_state.read().is_syncing(), - is_optimistic: Some(is_optimistic), - el_offline: Some(el_offline), - head_slot, - sync_distance, - }; + let syncing_data = api_types::SyncingData { + is_syncing: network_globals.sync_state.read().is_syncing(), + is_optimistic: Some(is_optimistic), + el_offline: Some(el_offline), + head_slot, + sync_distance, + }; - Ok(api_types::GenericResponse::from(syncing_data)) - }) - .await + Ok(api_types::GenericResponse::from(syncing_data)) + }) + .await } }, ); @@ -2592,10 +2735,13 @@ pub fn serve( .and(warp::path("node")) .and(warp::path("health")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(network_globals.clone()) .and(chain_filter.clone()) .and_then( - |network_globals: Arc>, chain: Arc>| { + |task_spawner: TaskSpawner, + network_globals: Arc>, + chain: Arc>| { async move { let el_offline = if let Some(el) = &chain.execution_layer { el.is_offline_or_erroring().await @@ -2603,28 +2749,31 @@ pub fn serve( true }; - blocking_response_task(move || { - let is_optimistic = chain - .is_optimistic_or_invalid_head() - .map_err(warp_utils::reject::beacon_chain_error)?; + task_spawner + .blocking_response_task(Priority::P0, move || { + let is_optimistic = chain + .is_optimistic_or_invalid_head() + .map_err(warp_utils::reject::beacon_chain_error)?; - let is_syncing = !network_globals.sync_state.read().is_synced(); + let is_syncing = !network_globals.sync_state.read().is_synced(); - if el_offline { - Err(warp_utils::reject::not_synced("execution layer is offline".to_string())) - } else if is_syncing || is_optimistic { - Ok(warp::reply::with_status( - warp::reply(), - warp::http::StatusCode::PARTIAL_CONTENT, - )) - } else { - Ok(warp::reply::with_status( - warp::reply(), - warp::http::StatusCode::OK, - )) - } - }) - .await + if el_offline { + Err(warp_utils::reject::not_synced( + "execution layer is offline".to_string(), + )) + } else if is_syncing || is_optimistic { + Ok(warp::reply::with_status( + warp::reply(), + warp::http::StatusCode::PARTIAL_CONTENT, + )) + } else { + Ok(warp::reply::with_status( + warp::reply(), + warp::http::StatusCode::OK, + )) + } + }) + .await } }, ); @@ -2635,10 +2784,13 @@ pub fn serve( .and(warp::path("peers")) .and(warp::path::param::()) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(network_globals.clone()) .and_then( - |requested_peer_id: String, network_globals: Arc>| { - blocking_json_task(move || { + |requested_peer_id: String, + task_spawner: TaskSpawner, + network_globals: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { let peer_id = PeerId::from_bytes( &bs58::decode(requested_peer_id.as_str()) .into_vec() @@ -2692,11 +2844,13 @@ pub fn serve( .and(warp::path("peers")) .and(warp::path::end()) .and(multi_key_query::()) + .and(task_spawner_filter.clone()) .and(network_globals.clone()) .and_then( |query_res: Result, + task_spawner: TaskSpawner, network_globals: Arc>| { - blocking_json_task(move || { + task_spawner.blocking_json_task(Priority::P1, move || { let query = query_res?; let mut peers: Vec = Vec::new(); network_globals @@ -2760,38 +2914,42 @@ pub fn serve( .and(warp::path("node")) .and(warp::path("peer_count")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(network_globals.clone()) - .and_then(|network_globals: Arc>| { - blocking_json_task(move || { - let mut connected: u64 = 0; - let mut connecting: u64 = 0; - let mut disconnected: u64 = 0; - let mut disconnecting: u64 = 0; - - network_globals - .peers - .read() - .peers() - .for_each(|(_, peer_info)| { - let state = api_types::PeerState::from_peer_connection_status( - peer_info.connection_status(), - ); - match state { - api_types::PeerState::Connected => connected += 1, - api_types::PeerState::Connecting => connecting += 1, - api_types::PeerState::Disconnected => disconnected += 1, - api_types::PeerState::Disconnecting => disconnecting += 1, - } - }); - - Ok(api_types::GenericResponse::from(api_types::PeerCount { - connected, - connecting, - disconnected, - disconnecting, - })) - }) - }); + .and_then( + |task_spawner: TaskSpawner, + network_globals: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + let mut connected: u64 = 0; + let mut connecting: u64 = 0; + let mut disconnected: u64 = 0; + let mut disconnecting: u64 = 0; + + network_globals + .peers + .read() + .peers() + .for_each(|(_, peer_info)| { + let state = api_types::PeerState::from_peer_connection_status( + peer_info.connection_status(), + ); + match state { + api_types::PeerState::Connected => connected += 1, + api_types::PeerState::Connecting => connecting += 1, + api_types::PeerState::Disconnected => disconnected += 1, + api_types::PeerState::Disconnecting => disconnecting += 1, + } + }); + + Ok(api_types::GenericResponse::from(api_types::PeerCount { + connected, + connecting, + disconnected, + disconnecting, + })) + }) + }, + ); /* * validator */ @@ -2808,11 +2966,19 @@ pub fn serve( })) .and(warp::path::end()) .and(not_while_syncing_filter.clone()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(log_filter.clone()) - .and_then(|epoch: Epoch, chain: Arc>, log: Logger| { - blocking_json_task(move || proposer_duties::proposer_duties(epoch, &chain, &log)) - }); + .and_then( + |epoch: Epoch, + task_spawner: TaskSpawner, + chain: Arc>, + log: Logger| { + task_spawner.blocking_json_task(Priority::P0, move || { + proposer_duties::proposer_duties(epoch, &chain, &log) + }) + }, + ); // GET validator/blocks/{slot} let get_validator_blocks = any_version @@ -2826,57 +2992,61 @@ pub fn serve( .and(warp::path::end()) .and(not_while_syncing_filter.clone()) .and(warp::query::()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(log_filter.clone()) .and_then( |endpoint_version: EndpointVersion, slot: Slot, query: api_types::ValidatorBlocksQuery, + task_spawner: TaskSpawner, chain: Arc>, - log: Logger| async move { - debug!( - log, - "Block production request from HTTP API"; - "slot" => slot - ); + log: Logger| { + task_spawner.spawn_async_with_rejection(Priority::P0, async move { + debug!( + log, + "Block production request from HTTP API"; + "slot" => slot + ); - let randao_reveal = query.randao_reveal.decompress().map_err(|e| { - warp_utils::reject::custom_bad_request(format!( - "randao reveal is not a valid BLS signature: {:?}", - e - )) - })?; + let randao_reveal = query.randao_reveal.decompress().map_err(|e| { + warp_utils::reject::custom_bad_request(format!( + "randao reveal is not a valid BLS signature: {:?}", + e + )) + })?; - let randao_verification = - if query.skip_randao_verification == SkipRandaoVerification::Yes { - if !randao_reveal.is_infinity() { - return Err(warp_utils::reject::custom_bad_request( + let randao_verification = + if query.skip_randao_verification == SkipRandaoVerification::Yes { + if !randao_reveal.is_infinity() { + return Err(warp_utils::reject::custom_bad_request( "randao_reveal must be point-at-infinity if verification is skipped" .into(), )); - } - ProduceBlockVerification::NoVerification - } else { - ProduceBlockVerification::VerifyRandao - }; + } + ProduceBlockVerification::NoVerification + } else { + ProduceBlockVerification::VerifyRandao + }; - let (block, _) = chain - .produce_block_with_verification::>( - randao_reveal, - slot, - query.graffiti.map(Into::into), - randao_verification, - ) - .await - .map_err(warp_utils::reject::block_production_error)?; - let fork_name = block - .to_ref() - .fork_name(&chain.spec) - .map_err(inconsistent_fork_rejection)?; - - fork_versioned_response(endpoint_version, fork_name, block) - .map(|response| warp::reply::json(&response).into_response()) - .map(|res| add_consensus_version_header(res, fork_name)) + let (block, _) = chain + .produce_block_with_verification::>( + randao_reveal, + slot, + query.graffiti.map(Into::into), + randao_verification, + ) + .await + .map_err(warp_utils::reject::block_production_error)?; + let fork_name = block + .to_ref() + .fork_name(&chain.spec) + .map_err(inconsistent_fork_rejection)?; + + fork_versioned_response(endpoint_version, fork_name, block) + .map(|response| warp::reply::json(&response).into_response()) + .map(|res| add_consensus_version_header(res, fork_name)) + }) }, ); @@ -2892,49 +3062,53 @@ pub fn serve( .and(warp::path::end()) .and(not_while_syncing_filter.clone()) .and(warp::query::()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and_then( |slot: Slot, query: api_types::ValidatorBlocksQuery, - chain: Arc>| async move { - let randao_reveal = query.randao_reveal.decompress().map_err(|e| { - warp_utils::reject::custom_bad_request(format!( - "randao reveal is not a valid BLS signature: {:?}", - e - )) - })?; + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.spawn_async_with_rejection(Priority::P0, async move { + let randao_reveal = query.randao_reveal.decompress().map_err(|e| { + warp_utils::reject::custom_bad_request(format!( + "randao reveal is not a valid BLS signature: {:?}", + e + )) + })?; - let randao_verification = - if query.skip_randao_verification == SkipRandaoVerification::Yes { - if !randao_reveal.is_infinity() { - return Err(warp_utils::reject::custom_bad_request( + let randao_verification = + if query.skip_randao_verification == SkipRandaoVerification::Yes { + if !randao_reveal.is_infinity() { + return Err(warp_utils::reject::custom_bad_request( "randao_reveal must be point-at-infinity if verification is skipped" .into() )); - } - ProduceBlockVerification::NoVerification - } else { - ProduceBlockVerification::VerifyRandao - }; + } + ProduceBlockVerification::NoVerification + } else { + ProduceBlockVerification::VerifyRandao + }; - let (block, _) = chain - .produce_block_with_verification::>( - randao_reveal, - slot, - query.graffiti.map(Into::into), - randao_verification, - ) - .await - .map_err(warp_utils::reject::block_production_error)?; - let fork_name = block - .to_ref() - .fork_name(&chain.spec) - .map_err(inconsistent_fork_rejection)?; - - // Pose as a V2 endpoint so we return the fork `version`. - fork_versioned_response(V2, fork_name, block) - .map(|response| warp::reply::json(&response).into_response()) - .map(|res| add_consensus_version_header(res, fork_name)) + let (block, _) = chain + .produce_block_with_verification::>( + randao_reveal, + slot, + query.graffiti.map(Into::into), + randao_verification, + ) + .await + .map_err(warp_utils::reject::block_production_error)?; + let fork_name = block + .to_ref() + .fork_name(&chain.spec) + .map_err(inconsistent_fork_rejection)?; + + // Pose as a V2 endpoint so we return the fork `version`. + fork_versioned_response(V2, fork_name, block) + .map(|response| warp::reply::json(&response).into_response()) + .map(|res| add_consensus_version_header(res, fork_name)) + }) }, ); @@ -2945,10 +3119,13 @@ pub fn serve( .and(warp::path::end()) .and(warp::query::()) .and(not_while_syncing_filter.clone()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and_then( - |query: api_types::ValidatorAttestationDataQuery, chain: Arc>| { - blocking_json_task(move || { + |query: api_types::ValidatorAttestationDataQuery, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P0, move || { let current_slot = chain .slot() .map_err(warp_utils::reject::beacon_chain_error)?; @@ -2977,10 +3154,13 @@ pub fn serve( .and(warp::path::end()) .and(warp::query::()) .and(not_while_syncing_filter.clone()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and_then( - |query: api_types::ValidatorAggregateAttestationQuery, chain: Arc>| { - blocking_json_task(move || { + |query: api_types::ValidatorAggregateAttestationQuery, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P0, move || { chain .get_aggregated_attestation_by_slot_and_root( query.slot, @@ -3015,10 +3195,14 @@ pub fn serve( .and(warp::path::end()) .and(not_while_syncing_filter.clone()) .and(warp::body::json()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and_then( - |epoch: Epoch, indices: api_types::ValidatorIndexData, chain: Arc>| { - blocking_json_task(move || { + |epoch: Epoch, + indices: api_types::ValidatorIndexData, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P0, move || { attester_duties::attester_duties(epoch, &indices.0, &chain) }) }, @@ -3037,10 +3221,14 @@ pub fn serve( .and(warp::path::end()) .and(not_while_syncing_filter.clone()) .and(warp::body::json()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and_then( - |epoch: Epoch, indices: api_types::ValidatorIndexData, chain: Arc>| { - blocking_json_task(move || { + |epoch: Epoch, + indices: api_types::ValidatorIndexData, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P0, move || { sync_committees::sync_committee_duties(epoch, &indices.0, &chain) }) }, @@ -3053,10 +3241,13 @@ pub fn serve( .and(warp::path::end()) .and(warp::query::()) .and(not_while_syncing_filter.clone()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and_then( - |sync_committee_data: SyncContributionData, chain: Arc>| { - blocking_json_task(move || { + |sync_committee_data: SyncContributionData, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P0, move || { chain .get_aggregated_sync_committee_contribution(&sync_committee_data) .map_err(|e| { @@ -3081,15 +3272,17 @@ pub fn serve( .and(warp::path("aggregate_and_proofs")) .and(warp::path::end()) .and(not_while_syncing_filter.clone()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(warp::body::json()) .and(network_tx_filter.clone()) .and(log_filter.clone()) .and_then( - |chain: Arc>, + |task_spawner: TaskSpawner, + chain: Arc>, aggregates: Vec>, network_tx: UnboundedSender>, log: Logger| { - blocking_json_task(move || { + task_spawner.blocking_json_task(Priority::P0, move || { let seen_timestamp = timestamp_now(); let mut verified_aggregates = Vec::with_capacity(aggregates.len()); let mut messages = Vec::with_capacity(aggregates.len()); @@ -3192,16 +3385,18 @@ pub fn serve( .and(warp::path("contribution_and_proofs")) .and(warp::path::end()) .and(not_while_syncing_filter.clone()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(warp::body::json()) .and(network_tx_filter) .and(log_filter.clone()) .and_then( - |chain: Arc>, + |task_spawner: TaskSpawner, + chain: Arc>, contributions: Vec>, network_tx: UnboundedSender>, log: Logger| { - blocking_json_task(move || { + task_spawner.blocking_json_task(Priority::P0, move || { sync_committees::process_signed_contribution_and_proofs( contributions, network_tx, @@ -3220,14 +3415,16 @@ pub fn serve( .and(warp::path::end()) .and(warp::body::json()) .and(validator_subscription_tx_filter.clone()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(log_filter.clone()) .and_then( |subscriptions: Vec, validator_subscription_tx: Sender, + task_spawner: TaskSpawner, chain: Arc>, log: Logger| { - blocking_json_task(move || { + task_spawner.blocking_json_task(Priority::P0, move || { for subscription in &subscriptions { chain .validator_monitor @@ -3269,45 +3466,49 @@ pub fn serve( .and(warp::path("prepare_beacon_proposer")) .and(warp::path::end()) .and(not_while_syncing_filter.clone()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(log_filter.clone()) .and(warp::body::json()) .and_then( - |chain: Arc>, + |task_spawner: TaskSpawner, + chain: Arc>, log: Logger, - preparation_data: Vec| async move { - let execution_layer = chain - .execution_layer - .as_ref() - .ok_or(BeaconChainError::ExecutionLayerMissing) - .map_err(warp_utils::reject::beacon_chain_error)?; - - let current_slot = chain - .slot() - .map_err(warp_utils::reject::beacon_chain_error)?; - let current_epoch = current_slot.epoch(T::EthSpec::slots_per_epoch()); + preparation_data: Vec| { + task_spawner.spawn_async_with_rejection(Priority::P0, async move { + let execution_layer = chain + .execution_layer + .as_ref() + .ok_or(BeaconChainError::ExecutionLayerMissing) + .map_err(warp_utils::reject::beacon_chain_error)?; - debug!( - log, - "Received proposer preparation data"; - "count" => preparation_data.len(), - ); + let current_slot = chain + .slot() + .map_err(warp_utils::reject::beacon_chain_error)?; + let current_epoch = current_slot.epoch(T::EthSpec::slots_per_epoch()); - execution_layer - .update_proposer_preparation(current_epoch, &preparation_data) - .await; + debug!( + log, + "Received proposer preparation data"; + "count" => preparation_data.len(), + ); - chain - .prepare_beacon_proposer(current_slot) - .await - .map_err(|e| { - warp_utils::reject::custom_bad_request(format!( - "error updating proposer preparations: {:?}", - e - )) - })?; + execution_layer + .update_proposer_preparation(current_epoch, &preparation_data) + .await; + + chain + .prepare_beacon_proposer(current_slot) + .await + .map_err(|e| { + warp_utils::reject::custom_bad_request(format!( + "error updating proposer preparations: {:?}", + e + )) + })?; - Ok::<_, warp::reject::Rejection>(warp::reply::json(&()).into_response()) + Ok::<_, warp::reject::Rejection>(warp::reply::json(&()).into_response()) + }) }, ); @@ -3316,127 +3517,171 @@ pub fn serve( .and(warp::path("validator")) .and(warp::path("register_validator")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(log_filter.clone()) .and(warp::body::json()) .and_then( - |chain: Arc>, + |task_spawner: TaskSpawner, + chain: Arc>, log: Logger, - register_val_data: Vec| async move { - let execution_layer = chain - .execution_layer - .as_ref() - .ok_or(BeaconChainError::ExecutionLayerMissing) - .map_err(warp_utils::reject::beacon_chain_error)?; - let current_slot = chain - .slot_clock - .now_or_genesis() - .ok_or(BeaconChainError::UnableToReadSlot) - .map_err(warp_utils::reject::beacon_chain_error)?; - let current_epoch = current_slot.epoch(T::EthSpec::slots_per_epoch()); + register_val_data: Vec| async { + let (tx, rx) = oneshot::channel(); + + task_spawner + .spawn_async_with_rejection(Priority::P0, async move { + let execution_layer = chain + .execution_layer + .as_ref() + .ok_or(BeaconChainError::ExecutionLayerMissing) + .map_err(warp_utils::reject::beacon_chain_error)?; + let current_slot = chain + .slot_clock + .now_or_genesis() + .ok_or(BeaconChainError::UnableToReadSlot) + .map_err(warp_utils::reject::beacon_chain_error)?; + let current_epoch = current_slot.epoch(T::EthSpec::slots_per_epoch()); - debug!( - log, - "Received register validator request"; - "count" => register_val_data.len(), - ); + debug!( + log, + "Received register validator request"; + "count" => register_val_data.len(), + ); - let head_snapshot = chain.head_snapshot(); - let spec = &chain.spec; + let head_snapshot = chain.head_snapshot(); + let spec = &chain.spec; - let (preparation_data, filtered_registration_data): ( - Vec, - Vec, - ) = register_val_data - .into_iter() - .filter_map(|register_data| { - chain - .validator_index(®ister_data.message.pubkey) - .ok() - .flatten() - .and_then(|validator_index| { - let validator = head_snapshot - .beacon_state - .get_validator(validator_index) - .ok()?; - let validator_status = ValidatorStatus::from_validator( - validator, - current_epoch, - spec.far_future_epoch, - ) - .superstatus(); - let is_active_or_pending = - matches!(validator_status, ValidatorStatus::Pending) - || matches!(validator_status, ValidatorStatus::Active); - - // Filter out validators who are not 'active' or 'pending'. - is_active_or_pending.then_some({ - ( - ProposerPreparationData { - validator_index: validator_index as u64, - fee_recipient: register_data.message.fee_recipient, - }, - register_data, - ) - }) + let (preparation_data, filtered_registration_data): ( + Vec, + Vec, + ) = register_val_data + .into_iter() + .filter_map(|register_data| { + chain + .validator_index(®ister_data.message.pubkey) + .ok() + .flatten() + .and_then(|validator_index| { + let validator = head_snapshot + .beacon_state + .get_validator(validator_index) + .ok()?; + let validator_status = ValidatorStatus::from_validator( + validator, + current_epoch, + spec.far_future_epoch, + ) + .superstatus(); + let is_active_or_pending = + matches!(validator_status, ValidatorStatus::Pending) + || matches!( + validator_status, + ValidatorStatus::Active + ); + + // Filter out validators who are not 'active' or 'pending'. + is_active_or_pending.then_some({ + ( + ProposerPreparationData { + validator_index: validator_index as u64, + fee_recipient: register_data + .message + .fee_recipient, + }, + register_data, + ) + }) + }) }) - }) - .unzip(); - - // Update the prepare beacon proposer cache based on this request. - execution_layer - .update_proposer_preparation(current_epoch, &preparation_data) - .await; - - // Call prepare beacon proposer blocking with the latest update in order to make - // sure we have a local payload to fall back to in the event of the blinded block - // flow failing. - chain - .prepare_beacon_proposer(current_slot) - .await - .map_err(|e| { - warp_utils::reject::custom_bad_request(format!( - "error updating proposer preparations: {:?}", - e - )) - })?; + .unzip(); - let builder = execution_layer - .builder() - .as_ref() - .ok_or(BeaconChainError::BuilderMissing) - .map_err(warp_utils::reject::beacon_chain_error)?; + // Update the prepare beacon proposer cache based on this request. + execution_layer + .update_proposer_preparation(current_epoch, &preparation_data) + .await; - info!( - log, - "Forwarding register validator request to connected builder"; - "count" => filtered_registration_data.len(), - ); + // Call prepare beacon proposer blocking with the latest update in order to make + // sure we have a local payload to fall back to in the event of the blinded block + // flow failing. + chain + .prepare_beacon_proposer(current_slot) + .await + .map_err(|e| { + warp_utils::reject::custom_bad_request(format!( + "error updating proposer preparations: {:?}", + e + )) + })?; - builder - .post_builder_validators(&filtered_registration_data) - .await - .map(|resp| warp::reply::json(&resp).into_response()) - .map_err(|e| { - warn!( + info!( log, - "Relay error when registering validator(s)"; - "num_registrations" => filtered_registration_data.len(), - "error" => ?e + "Forwarding register validator request to connected builder"; + "count" => filtered_registration_data.len(), ); - // Forward the HTTP status code if we are able to, otherwise fall back - // to a server error. - if let eth2::Error::ServerMessage(message) = e { - if message.code == StatusCode::BAD_REQUEST.as_u16() { - return warp_utils::reject::custom_bad_request(message.message); - } else { - // According to the spec this response should only be a 400 or 500, - // so we fall back to a 500 here. - return warp_utils::reject::custom_server_error(message.message); - } - } - warp_utils::reject::custom_server_error(format!("{e:?}")) + + // It's a waste of a `BeaconProcessor` worker to just + // wait on a response from the builder (especially since + // they have frequent timeouts). Spawn a new task and + // send the response back to our original HTTP request + // task via a channel. + let builder_future = async move { + let builder = chain + .execution_layer + .as_ref() + .ok_or(BeaconChainError::ExecutionLayerMissing) + .map_err(warp_utils::reject::beacon_chain_error)? + .builder() + .as_ref() + .ok_or(BeaconChainError::BuilderMissing) + .map_err(warp_utils::reject::beacon_chain_error)?; + + builder + .post_builder_validators(&filtered_registration_data) + .await + .map(|resp| warp::reply::json(&resp).into_response()) + .map_err(|e| { + warn!( + log, + "Relay error when registering validator(s)"; + "num_registrations" => filtered_registration_data.len(), + "error" => ?e + ); + // Forward the HTTP status code if we are able to, otherwise fall back + // to a server error. + if let eth2::Error::ServerMessage(message) = e { + if message.code == StatusCode::BAD_REQUEST.as_u16() { + return warp_utils::reject::custom_bad_request( + message.message, + ); + } else { + // According to the spec this response should only be a 400 or 500, + // so we fall back to a 500 here. + return warp_utils::reject::custom_server_error( + message.message, + ); + } + } + warp_utils::reject::custom_server_error(format!("{e:?}")) + }) + }; + tokio::task::spawn(async move { tx.send(builder_future.await) }); + + // Just send a generic 200 OK from this closure. We'll + // ignore the `Ok` variant and form a proper response + // from what is sent back down the channel. + Ok(warp::reply::reply().into_response()) }) + .await?; + + // Await a response from the builder without blocking a + // `BeaconProcessor` worker. + rx.await.unwrap_or_else(|_| { + Ok(warp::reply::with_status( + warp::reply::json(&"No response from channel"), + eth2::StatusCode::INTERNAL_SERVER_ERROR, + ) + .into_response()) + }) }, ); // POST validator/sync_committee_subscriptions @@ -3446,15 +3691,17 @@ pub fn serve( .and(warp::path::end()) .and(warp::body::json()) .and(validator_subscription_tx_filter) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(log_filter.clone()) .and_then( |subscriptions: Vec, validator_subscription_tx: Sender, + task_spawner: TaskSpawner, chain: Arc>, log: Logger | { - blocking_json_task(move || { + task_spawner.blocking_json_task(Priority::P0, move || { for subscription in subscriptions { chain .validator_monitor @@ -3489,10 +3736,14 @@ pub fn serve( .and(warp::path::param::()) .and(warp::path::end()) .and(warp::body::json()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and_then( - |epoch: Epoch, indices: Vec, chain: Arc>| { - blocking_json_task(move || { + |epoch: Epoch, + indices: Vec, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P0, move || { // Ensure the request is for either the current, previous or next epoch. let current_epoch = chain .epoch() @@ -3526,10 +3777,13 @@ pub fn serve( .and(warp::path("liveness")) .and(warp::path::end()) .and(warp::body::json()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and_then( - |request_data: api_types::LivenessRequestData, chain: Arc>| { - blocking_json_task(move || { + |request_data: api_types::LivenessRequestData, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P0, move || { // Ensure the request is for either the current, previous or next epoch. let current_epoch = chain .epoch() @@ -3568,8 +3822,9 @@ pub fn serve( let get_lighthouse_health = warp::path("lighthouse") .and(warp::path("health")) .and(warp::path::end()) - .and_then(|| { - blocking_json_task(move || { + .and(task_spawner_filter.clone()) + .and_then(|task_spawner: TaskSpawner| { + task_spawner.blocking_json_task(Priority::P0, move || { eth2::lighthouse::Health::observe() .map(api_types::GenericResponse::from) .map_err(warp_utils::reject::custom_bad_request) @@ -3581,13 +3836,18 @@ pub fn serve( .and(warp::path("ui")) .and(warp::path("health")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(system_info_filter) .and(app_start_filter) .and(data_dir_filter) .and(network_globals.clone()) .and_then( - |sysinfo, app_start: std::time::Instant, data_dir, network_globals| { - blocking_json_task(move || { + |task_spawner: TaskSpawner, + sysinfo, + app_start: std::time::Instant, + data_dir, + network_globals| { + task_spawner.blocking_json_task(Priority::P0, move || { let app_uptime = app_start.elapsed().as_secs(); Ok(api_types::GenericResponse::from(observe_system_health_bn( sysinfo, @@ -3604,12 +3864,15 @@ pub fn serve( .and(warp::path("ui")) .and(warp::path("validator_count")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(|chain: Arc>| { - blocking_json_task(move || { - ui::get_validator_count(chain).map(api_types::GenericResponse::from) - }) - }); + .and_then( + |task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + ui::get_validator_count(chain).map(api_types::GenericResponse::from) + }) + }, + ); // POST lighthouse/ui/validator_metrics let post_lighthouse_ui_validator_metrics = warp::path("lighthouse") @@ -3617,10 +3880,13 @@ pub fn serve( .and(warp::path("validator_metrics")) .and(warp::path::end()) .and(warp::body::json()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and_then( - |request_data: ui::ValidatorMetricsRequestData, chain: Arc>| { - blocking_json_task(move || { + |request_data: ui::ValidatorMetricsRequestData, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { ui::post_validator_monitor_metrics(request_data, chain) .map(api_types::GenericResponse::from) }) @@ -3633,10 +3899,13 @@ pub fn serve( .and(warp::path("validator_info")) .and(warp::path::end()) .and(warp::body::json()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and_then( - |request_data: ui::ValidatorInfoRequestData, chain: Arc>| { - blocking_json_task(move || { + |request_data: ui::ValidatorInfoRequestData, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { ui::get_validator_info(request_data, chain) .map(api_types::GenericResponse::from) }) @@ -3647,21 +3916,26 @@ pub fn serve( let get_lighthouse_syncing = warp::path("lighthouse") .and(warp::path("syncing")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(network_globals.clone()) - .and_then(|network_globals: Arc>| { - blocking_json_task(move || { - Ok(api_types::GenericResponse::from( - network_globals.sync_state(), - )) - }) - }); + .and_then( + |task_spawner: TaskSpawner, + network_globals: Arc>| { + task_spawner.blocking_json_task(Priority::P0, move || { + Ok(api_types::GenericResponse::from( + network_globals.sync_state(), + )) + }) + }, + ); // GET lighthouse/nat let get_lighthouse_nat = warp::path("lighthouse") .and(warp::path("nat")) + .and(task_spawner_filter.clone()) .and(warp::path::end()) - .and_then(|| { - blocking_json_task(move || { + .and_then(|task_spawner: TaskSpawner| { + task_spawner.blocking_json_task(Priority::P1, move || { Ok(api_types::GenericResponse::from( lighthouse_network::metrics::NAT_OPEN .as_ref() @@ -3676,57 +3950,70 @@ pub fn serve( let get_lighthouse_peers = warp::path("lighthouse") .and(warp::path("peers")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(network_globals.clone()) - .and_then(|network_globals: Arc>| { - blocking_json_task(move || { - Ok(network_globals - .peers - .read() - .peers() - .map(|(peer_id, peer_info)| eth2::lighthouse::Peer { - peer_id: peer_id.to_string(), - peer_info: peer_info.clone(), - }) - .collect::>()) - }) - }); + .and_then( + |task_spawner: TaskSpawner, + network_globals: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + Ok(network_globals + .peers + .read() + .peers() + .map(|(peer_id, peer_info)| eth2::lighthouse::Peer { + peer_id: peer_id.to_string(), + peer_info: peer_info.clone(), + }) + .collect::>()) + }) + }, + ); // GET lighthouse/peers/connected let get_lighthouse_peers_connected = warp::path("lighthouse") .and(warp::path("peers")) .and(warp::path("connected")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(network_globals) - .and_then(|network_globals: Arc>| { - blocking_json_task(move || { - Ok(network_globals - .peers - .read() - .connected_peers() - .map(|(peer_id, peer_info)| eth2::lighthouse::Peer { - peer_id: peer_id.to_string(), - peer_info: peer_info.clone(), - }) - .collect::>()) - }) - }); + .and_then( + |task_spawner: TaskSpawner, + network_globals: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + Ok(network_globals + .peers + .read() + .connected_peers() + .map(|(peer_id, peer_info)| eth2::lighthouse::Peer { + peer_id: peer_id.to_string(), + peer_info: peer_info.clone(), + }) + .collect::>()) + }) + }, + ); // GET lighthouse/proto_array let get_lighthouse_proto_array = warp::path("lighthouse") .and(warp::path("proto_array")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(|chain: Arc>| { - blocking_response_task(move || { - Ok::<_, warp::Rejection>(warp::reply::json(&api_types::GenericResponseRef::from( - chain - .canonical_head - .fork_choice_read_lock() - .proto_array() - .core_proto_array(), - ))) - }) - }); + .and_then( + |task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_response_task(Priority::P1, move || { + Ok::<_, warp::Rejection>(warp::reply::json( + &api_types::GenericResponseRef::from( + chain + .canonical_head + .fork_choice_read_lock() + .proto_array() + .core_proto_array(), + ), + )) + }) + }, + ); // GET lighthouse/validator_inclusion/{epoch}/{validator_id} let get_lighthouse_validator_inclusion_global = warp::path("lighthouse") @@ -3734,10 +4021,14 @@ pub fn serve( .and(warp::path::param::()) .and(warp::path::param::()) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and_then( - |epoch: Epoch, validator_id: ValidatorId, chain: Arc>| { - blocking_json_task(move || { + |epoch: Epoch, + validator_id: ValidatorId, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { validator_inclusion::validator_inclusion_data(epoch, &validator_id, &chain) .map(api_types::GenericResponse::from) }) @@ -3750,82 +4041,94 @@ pub fn serve( .and(warp::path::param::()) .and(warp::path("global")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(|epoch: Epoch, chain: Arc>| { - blocking_json_task(move || { - validator_inclusion::global_validator_inclusion_data(epoch, &chain) - .map(api_types::GenericResponse::from) - }) - }); + .and_then( + |epoch: Epoch, task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + validator_inclusion::global_validator_inclusion_data(epoch, &chain) + .map(api_types::GenericResponse::from) + }) + }, + ); // GET lighthouse/eth1/syncing let get_lighthouse_eth1_syncing = warp::path("lighthouse") .and(warp::path("eth1")) .and(warp::path("syncing")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(|chain: Arc>| { - blocking_json_task(move || { - let current_slot_opt = chain.slot().ok(); - - chain - .eth1_chain - .as_ref() - .ok_or_else(|| { - warp_utils::reject::custom_not_found( - "Eth1 sync is disabled. See the --eth1 CLI flag.".to_string(), - ) - }) - .and_then(|eth1| { - eth1.sync_status(chain.genesis_time, current_slot_opt, &chain.spec) - .ok_or_else(|| { - warp_utils::reject::custom_server_error( - "Unable to determine Eth1 sync status".to_string(), - ) - }) - }) - .map(api_types::GenericResponse::from) - }) - }); + .and_then( + |task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + let current_slot_opt = chain.slot().ok(); + + chain + .eth1_chain + .as_ref() + .ok_or_else(|| { + warp_utils::reject::custom_not_found( + "Eth1 sync is disabled. See the --eth1 CLI flag.".to_string(), + ) + }) + .and_then(|eth1| { + eth1.sync_status(chain.genesis_time, current_slot_opt, &chain.spec) + .ok_or_else(|| { + warp_utils::reject::custom_server_error( + "Unable to determine Eth1 sync status".to_string(), + ) + }) + }) + .map(api_types::GenericResponse::from) + }) + }, + ); // GET lighthouse/eth1/block_cache let get_lighthouse_eth1_block_cache = warp::path("lighthouse") .and(warp::path("eth1")) .and(warp::path("block_cache")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(eth1_service_filter.clone()) - .and_then(|eth1_service: eth1::Service| { - blocking_json_task(move || { - Ok(api_types::GenericResponse::from( - eth1_service - .blocks() - .read() - .iter() - .cloned() - .collect::>(), - )) - }) - }); + .and_then( + |task_spawner: TaskSpawner, eth1_service: eth1::Service| { + task_spawner.blocking_json_task(Priority::P1, move || { + Ok(api_types::GenericResponse::from( + eth1_service + .blocks() + .read() + .iter() + .cloned() + .collect::>(), + )) + }) + }, + ); // GET lighthouse/eth1/deposit_cache let get_lighthouse_eth1_deposit_cache = warp::path("lighthouse") .and(warp::path("eth1")) .and(warp::path("deposit_cache")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(eth1_service_filter) - .and_then(|eth1_service: eth1::Service| { - blocking_json_task(move || { - Ok(api_types::GenericResponse::from( - eth1_service - .deposits() - .read() - .cache - .iter() - .cloned() - .collect::>(), - )) - }) - }); + .and_then( + |task_spawner: TaskSpawner, eth1_service: eth1::Service| { + task_spawner.blocking_json_task(Priority::P1, move || { + Ok(api_types::GenericResponse::from( + eth1_service + .deposits() + .read() + .cache + .iter() + .cloned() + .collect::>(), + )) + }) + }, + ); // GET lighthouse/beacon/states/{state_id}/ssz let get_lighthouse_beacon_states_ssz = warp::path("lighthouse") @@ -3834,42 +4137,50 @@ pub fn serve( .and(warp::path::param::()) .and(warp::path("ssz")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(|state_id: StateId, chain: Arc>| { - blocking_response_task(move || { - // This debug endpoint provides no indication of optimistic status. - let (state, _execution_optimistic, _finalized) = state_id.state(&chain)?; - Response::builder() - .status(200) - .header("Content-Type", "application/ssz") - .body(state.as_ssz_bytes()) - .map_err(|e| { - warp_utils::reject::custom_server_error(format!( - "failed to create response: {}", - e - )) - }) - }) - }); + .and_then( + |state_id: StateId, + task_spawner: TaskSpawner, + chain: Arc>| { + task_spawner.blocking_response_task(Priority::P1, move || { + // This debug endpoint provides no indication of optimistic status. + let (state, _execution_optimistic, _finalized) = state_id.state(&chain)?; + Response::builder() + .status(200) + .header("Content-Type", "application/ssz") + .body(state.as_ssz_bytes()) + .map_err(|e| { + warp_utils::reject::custom_server_error(format!( + "failed to create response: {}", + e + )) + }) + }) + }, + ); // GET lighthouse/staking let get_lighthouse_staking = warp::path("lighthouse") .and(warp::path("staking")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(|chain: Arc>| { - blocking_json_task(move || { - if chain.eth1_chain.is_some() { - Ok(()) - } else { - Err(warp_utils::reject::custom_not_found( - "staking is not enabled, \ + .and_then( + |task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + if chain.eth1_chain.is_some() { + Ok(()) + } else { + Err(warp_utils::reject::custom_not_found( + "staking is not enabled, \ see the --staking CLI flag" - .to_string(), - )) - } - }) - }); + .to_string(), + )) + } + }) + }, + ); let database_path = warp::path("lighthouse").and(warp::path("database")); @@ -3877,31 +4188,41 @@ pub fn serve( let get_lighthouse_database_info = database_path .and(warp::path("info")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(|chain: Arc>| blocking_json_task(move || database::info(chain))); + .and_then( + |task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || database::info(chain)) + }, + ); // POST lighthouse/database/reconstruct let post_lighthouse_database_reconstruct = database_path .and(warp::path("reconstruct")) .and(warp::path::end()) .and(not_while_syncing_filter) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(|chain: Arc>| { - blocking_json_task(move || { - chain.store_migrator.process_reconstruction(); - Ok("success") - }) - }); + .and_then( + |task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + chain.store_migrator.process_reconstruction(); + Ok("success") + }) + }, + ); // POST lighthouse/database/historical_blocks let post_lighthouse_database_historical_blocks = database_path .and(warp::path("historical_blocks")) .and(warp::path::end()) .and(warp::body::json()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(log_filter.clone()) .and_then( |blocks: Vec>>, + task_spawner: TaskSpawner, chain: Arc>, log: Logger| { info!( @@ -3910,7 +4231,9 @@ pub fn serve( "count" => blocks.len(), "source" => "http_api" ); - blocking_json_task(move || database::historical_blocks(chain, blocks)) + task_spawner.blocking_json_task(Priority::P1, move || { + database::historical_blocks(chain, blocks) + }) }, ); @@ -3920,10 +4243,13 @@ pub fn serve( .and(warp::path("block_rewards")) .and(warp::query::()) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(log_filter.clone()) - .and_then(|query, chain, log| { - blocking_json_task(move || block_rewards::get_block_rewards(query, chain, log)) + .and_then(|query, task_spawner: TaskSpawner, chain, log| { + task_spawner.blocking_json_task(Priority::P1, move || { + block_rewards::get_block_rewards(query, chain, log) + }) }); // POST lighthouse/analysis/block_rewards @@ -3932,11 +4258,16 @@ pub fn serve( .and(warp::path("block_rewards")) .and(warp::body::json()) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) .and(log_filter.clone()) - .and_then(|blocks, chain, log| { - blocking_json_task(move || block_rewards::compute_block_rewards(blocks, chain, log)) - }); + .and_then( + |blocks, task_spawner: TaskSpawner, chain, log| { + task_spawner.blocking_json_task(Priority::P1, move || { + block_rewards::compute_block_rewards(blocks, chain, log) + }) + }, + ); // GET lighthouse/analysis/attestation_performance/{index} let get_lighthouse_attestation_performance = warp::path("lighthouse") @@ -3945,12 +4276,15 @@ pub fn serve( .and(warp::path::param::()) .and(warp::query::()) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(|target, query, chain: Arc>| { - blocking_json_task(move || { - attestation_performance::get_attestation_performance(target, query, chain) - }) - }); + .and_then( + |target, query, task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + attestation_performance::get_attestation_performance(target, query, chain) + }) + }, + ); // GET lighthouse/analysis/block_packing_efficiency let get_lighthouse_block_packing_efficiency = warp::path("lighthouse") @@ -3958,35 +4292,45 @@ pub fn serve( .and(warp::path("block_packing_efficiency")) .and(warp::query::()) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(|query, chain: Arc>| { - blocking_json_task(move || { - block_packing_efficiency::get_block_packing_efficiency(query, chain) - }) - }); + .and_then( + |query, task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.blocking_json_task(Priority::P1, move || { + block_packing_efficiency::get_block_packing_efficiency(query, chain) + }) + }, + ); // GET lighthouse/merge_readiness let get_lighthouse_merge_readiness = warp::path("lighthouse") .and(warp::path("merge_readiness")) .and(warp::path::end()) + .and(task_spawner_filter.clone()) .and(chain_filter.clone()) - .and_then(|chain: Arc>| async move { - let merge_readiness = chain.check_merge_readiness().await; - Ok::<_, warp::reject::Rejection>( - warp::reply::json(&api_types::GenericResponse::from(merge_readiness)) - .into_response(), - ) - }); + .and_then( + |task_spawner: TaskSpawner, chain: Arc>| { + task_spawner.spawn_async_with_rejection(Priority::P1, async move { + let merge_readiness = chain.check_merge_readiness().await; + Ok::<_, warp::reject::Rejection>( + warp::reply::json(&api_types::GenericResponse::from(merge_readiness)) + .into_response(), + ) + }) + }, + ); let get_events = eth_v1 .and(warp::path("events")) .and(warp::path::end()) .and(multi_key_query::()) + .and(task_spawner_filter.clone()) .and(chain_filter) .and_then( |topics_res: Result, + task_spawner: TaskSpawner, chain: Arc>| { - blocking_response_task(move || { + task_spawner.blocking_response_task(Priority::P0, move || { let topics = topics_res?; // for each topic subscribed spawn a new subscription let mut receivers = Vec::with_capacity(topics.topics.len()); @@ -4057,38 +4401,46 @@ pub fn serve( let lighthouse_log_events = warp::path("lighthouse") .and(warp::path("logs")) .and(warp::path::end()) + .and(task_spawner_filter) .and(sse_component_filter) - .and_then(|sse_component: Option| { - blocking_response_task(move || { - if let Some(logging_components) = sse_component { - // Build a JSON stream - let s = - BroadcastStream::new(logging_components.sender.subscribe()).map(|msg| { - match msg { - Ok(data) => { - // Serialize to json - match data.to_json_string() { - // Send the json as a Server Side Event - Ok(json) => Ok(Event::default().data(json)), - Err(e) => Err(warp_utils::reject::server_sent_event_error( - format!("Unable to serialize to JSON {}", e), - )), + .and_then( + |task_spawner: TaskSpawner, sse_component: Option| { + task_spawner.blocking_response_task(Priority::P1, move || { + if let Some(logging_components) = sse_component { + // Build a JSON stream + let s = BroadcastStream::new(logging_components.sender.subscribe()).map( + |msg| { + match msg { + Ok(data) => { + // Serialize to json + match data.to_json_string() { + // Send the json as a Server Side Event + Ok(json) => Ok(Event::default().data(json)), + Err(e) => { + Err(warp_utils::reject::server_sent_event_error( + format!("Unable to serialize to JSON {}", e), + )) + } + } } + Err(e) => Err(warp_utils::reject::server_sent_event_error( + format!("Unable to receive event {}", e), + )), } - Err(e) => Err(warp_utils::reject::server_sent_event_error( - format!("Unable to receive event {}", e), - )), - } - }); + }, + ); - Ok::<_, warp::Rejection>(warp::sse::reply(warp::sse::keep_alive().stream(s))) - } else { - Err(warp_utils::reject::custom_server_error( - "SSE Logging is not enabled".to_string(), - )) - } - }) - }); + Ok::<_, warp::Rejection>(warp::sse::reply( + warp::sse::keep_alive().stream(s), + )) + } else { + Err(warp_utils::reject::custom_server_error( + "SSE Logging is not enabled".to_string(), + )) + } + }) + }, + ); // Define the ultimate set of routes that will be provided to the server. // Use `uor` rather than `or` in order to simplify types (see `UnifyingOrFilter`). diff --git a/beacon_node/http_api/src/task_spawner.rs b/beacon_node/http_api/src/task_spawner.rs new file mode 100644 index 00000000000..b4da67f77cf --- /dev/null +++ b/beacon_node/http_api/src/task_spawner.rs @@ -0,0 +1,214 @@ +use beacon_processor::{BeaconProcessorSend, BlockingOrAsync, Work, WorkEvent}; +use serde::Serialize; +use std::future::Future; +use tokio::sync::{mpsc::error::TrySendError, oneshot}; +use types::EthSpec; +use warp::reply::{Reply, Response}; + +/// Maps a request to a queue in the `BeaconProcessor`. +#[derive(Clone, Copy)] +pub enum Priority { + /// The highest priority. + P0, + /// The lowest priority. + P1, +} + +impl Priority { + /// Wrap `self` in a `WorkEvent` with an appropriate priority. + fn work_event(&self, process_fn: BlockingOrAsync) -> WorkEvent { + let work = match self { + Priority::P0 => Work::ApiRequestP0(process_fn), + Priority::P1 => Work::ApiRequestP1(process_fn), + }; + WorkEvent { + drop_during_sync: false, + work, + } + } +} + +/// Spawns tasks on the `BeaconProcessor` or directly on the tokio executor. +pub struct TaskSpawner { + /// Used to send tasks to the `BeaconProcessor`. The tokio executor will be + /// used if this is `None`. + beacon_processor_send: Option>, +} + +impl TaskSpawner { + pub fn new(beacon_processor_send: Option>) -> Self { + Self { + beacon_processor_send, + } + } + + /// Executes a "blocking" (non-async) task which returns a `Response`. + pub async fn blocking_response_task( + self, + priority: Priority, + func: F, + ) -> Result + where + F: FnOnce() -> Result + Send + Sync + 'static, + T: Reply + Send + 'static, + { + if let Some(beacon_processor_send) = &self.beacon_processor_send { + // Create a closure that will execute `func` and send the result to + // a channel held by this thread. + let (tx, rx) = oneshot::channel(); + let process_fn = move || { + // Execute the function, collect the return value. + let func_result = func(); + // Send the result down the channel. Ignore any failures; the + // send can only fail if the receiver is dropped. + let _ = tx.send(func_result); + }; + + // Send the function to the beacon processor for execution at some arbitrary time. + match send_to_beacon_processor( + beacon_processor_send, + priority, + BlockingOrAsync::Blocking(Box::new(process_fn)), + rx, + ) + .await + { + Ok(result) => result.map(Reply::into_response), + Err(error_response) => Ok(error_response), + } + } else { + // There is no beacon processor so spawn a task directly on the + // tokio executor. + warp_utils::task::blocking_response_task(func).await + } + } + + /// Executes a "blocking" (non-async) task which returns a JSON-serializable + /// object. + pub async fn blocking_json_task( + self, + priority: Priority, + func: F, + ) -> Result + where + F: FnOnce() -> Result + Send + Sync + 'static, + T: Serialize + Send + 'static, + { + let func = || func().map(|t| warp::reply::json(&t).into_response()); + self.blocking_response_task(priority, func).await + } + + /// Executes an async task which may return a `warp::Rejection`. + pub async fn spawn_async_with_rejection( + self, + priority: Priority, + func: impl Future> + Send + Sync + 'static, + ) -> Result { + if let Some(beacon_processor_send) = &self.beacon_processor_send { + // Create a wrapper future that will execute `func` and send the + // result to a channel held by this thread. + let (tx, rx) = oneshot::channel(); + let process_fn = async move { + // Await the future, collect the return value. + let func_result = func.await; + // Send the result down the channel. Ignore any failures; the + // send can only fail if the receiver is dropped. + let _ = tx.send(func_result); + }; + + // Send the function to the beacon processor for execution at some arbitrary time. + send_to_beacon_processor( + beacon_processor_send, + priority, + BlockingOrAsync::Async(Box::pin(process_fn)), + rx, + ) + .await + .unwrap_or_else(Result::Ok) + } else { + // There is no beacon processor so spawn a task directly on the + // tokio executor. + tokio::task::spawn(func).await.unwrap_or_else(|e| { + let response = warp::reply::with_status( + warp::reply::json(&format!("Tokio did not execute task: {e:?}")), + eth2::StatusCode::INTERNAL_SERVER_ERROR, + ) + .into_response(); + Ok(response) + }) + } + } + + /// Executes an async task which always returns a `Response`. + pub async fn spawn_async( + self, + priority: Priority, + func: impl Future + Send + Sync + 'static, + ) -> Response { + if let Some(beacon_processor_send) = &self.beacon_processor_send { + // Create a wrapper future that will execute `func` and send the + // result to a channel held by this thread. + let (tx, rx) = oneshot::channel(); + let process_fn = async move { + // Await the future, collect the return value. + let func_result = func.await; + // Send the result down the channel. Ignore any failures; the + // send can only fail if the receiver is dropped. + let _ = tx.send(func_result); + }; + + // Send the function to the beacon processor for execution at some arbitrary time. + send_to_beacon_processor( + beacon_processor_send, + priority, + BlockingOrAsync::Async(Box::pin(process_fn)), + rx, + ) + .await + .unwrap_or_else(|error_response| error_response) + } else { + // There is no beacon processor so spawn a task directly on the + // tokio executor. + tokio::task::spawn(func).await.unwrap_or_else(|e| { + warp::reply::with_status( + warp::reply::json(&format!("Tokio did not execute task: {e:?}")), + eth2::StatusCode::INTERNAL_SERVER_ERROR, + ) + .into_response() + }) + } + } +} + +/// Send a task to the beacon processor and await execution. +/// +/// If the task is not executed, return an `Err(response)` with an error message +/// for the API consumer. +async fn send_to_beacon_processor( + beacon_processor_send: &BeaconProcessorSend, + priority: Priority, + process_fn: BlockingOrAsync, + rx: oneshot::Receiver, +) -> Result { + let error_message = match beacon_processor_send.try_send(priority.work_event(process_fn)) { + Ok(()) => { + match rx.await { + // The beacon processor executed the task and sent a result. + Ok(func_result) => return Ok(func_result), + // The beacon processor dropped the channel without sending a + // result. The beacon processor dropped this task because its + // queues are full or it's shutting down. + Err(_) => "The task did not execute. The server is overloaded or shutting down.", + } + } + Err(TrySendError::Full(_)) => "The task was dropped. The server is overloaded.", + Err(TrySendError::Closed(_)) => "The task was dropped. The server is shutting down.", + }; + + let error_response = warp::reply::with_status( + warp::reply::json(&error_message), + eth2::StatusCode::INTERNAL_SERVER_ERROR, + ) + .into_response(); + Err(error_response) +} diff --git a/beacon_node/http_api/src/test_utils.rs b/beacon_node/http_api/src/test_utils.rs index 694402a3d78..0367776f8de 100644 --- a/beacon_node/http_api/src/test_utils.rs +++ b/beacon_node/http_api/src/test_utils.rs @@ -5,6 +5,7 @@ use beacon_chain::{ }, BeaconChain, BeaconChainTypes, }; +use beacon_processor::{BeaconProcessor, BeaconProcessorChannels, BeaconProcessorConfig}; use directory::DEFAULT_ROOT_DIR; use eth2::{BeaconNodeHttpClient, Timeouts}; use lighthouse_network::{ @@ -26,7 +27,7 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr}; use std::sync::Arc; use std::time::Duration; use store::MemoryStore; -use tokio::sync::oneshot; +use task_executor::test_utils::TestRuntime; use types::{ChainSpec, EthSpec}; pub const TCP_PORT: u16 = 42; @@ -39,7 +40,6 @@ pub struct InteractiveTester { pub harness: BeaconChainHarness>, pub client: BeaconNodeHttpClient, pub network_rx: NetworkReceivers, - _server_shutdown: oneshot::Sender<()>, } /// The result of calling `create_api_server`. @@ -48,7 +48,6 @@ pub struct InteractiveTester { pub struct ApiServer> { pub server: SFut, pub listening_socket: SocketAddr, - pub shutdown_tx: oneshot::Sender<()>, pub network_rx: NetworkReceivers, pub local_enr: Enr, pub external_peer_id: PeerId, @@ -96,10 +95,14 @@ impl InteractiveTester { let ApiServer { server, listening_socket, - shutdown_tx: _server_shutdown, network_rx, .. - } = create_api_server(harness.chain.clone(), harness.logger().clone()).await; + } = create_api_server( + harness.chain.clone(), + &harness.runtime, + harness.logger().clone(), + ) + .await; tokio::spawn(server); @@ -117,22 +120,23 @@ impl InteractiveTester { harness, client, network_rx, - _server_shutdown, } } } pub async fn create_api_server( chain: Arc>, + test_runtime: &TestRuntime, log: Logger, ) -> ApiServer> { // Get a random unused port. let port = unused_port::unused_tcp4_port().unwrap(); - create_api_server_on_port(chain, log, port).await + create_api_server_on_port(chain, test_runtime, log, port).await } pub async fn create_api_server_on_port( chain: Arc>, + test_runtime: &TestRuntime, log: Logger, port: u16, ) -> ApiServer> { @@ -180,6 +184,37 @@ pub async fn create_api_server_on_port( let eth1_service = eth1::Service::new(eth1::Config::default(), log.clone(), chain.spec.clone()).unwrap(); + let beacon_processor_config = BeaconProcessorConfig::default(); + let BeaconProcessorChannels { + beacon_processor_tx, + beacon_processor_rx, + work_reprocessing_tx, + work_reprocessing_rx, + } = BeaconProcessorChannels::new(&beacon_processor_config); + + let beacon_processor_send = beacon_processor_tx; + BeaconProcessor { + network_globals: network_globals.clone(), + executor: test_runtime.task_executor.clone(), + // The number of workers must be greater than one. Tests which use the + // builder workflow sometimes require an internal HTTP request in order + // to fulfill an already in-flight HTTP request, therefore having only + // one worker will result in a deadlock. + max_workers: 2, + current_workers: 0, + config: beacon_processor_config, + log: log.clone(), + } + .spawn_manager( + beacon_processor_rx, + work_reprocessing_tx, + work_reprocessing_rx, + None, + chain.slot_clock.clone(), + chain.spec.maximum_gossip_clock_disparity(), + ) + .unwrap(); + let ctx = Arc::new(Context { config: Config { enabled: true, @@ -190,26 +225,22 @@ pub async fn create_api_server_on_port( allow_sync_stalled: false, data_dir: std::path::PathBuf::from(DEFAULT_ROOT_DIR), spec_fork_name: None, + enable_beacon_processor: true, }, chain: Some(chain), network_senders: Some(network_senders), network_globals: Some(network_globals), + beacon_processor_send: Some(beacon_processor_send), eth1_service: Some(eth1_service), sse_logging_components: None, log, }); - let (shutdown_tx, shutdown_rx) = oneshot::channel(); - let server_shutdown = async { - // It's not really interesting why this triggered, just that it happened. - let _ = shutdown_rx.await; - }; - let (listening_socket, server) = crate::serve(ctx, server_shutdown).unwrap(); + let (listening_socket, server) = crate::serve(ctx, test_runtime.task_executor.exit()).unwrap(); ApiServer { server, listening_socket, - shutdown_tx, network_rx: network_receivers, local_enr: enr, external_peer_id: peer_id, diff --git a/beacon_node/http_api/src/validator.rs b/beacon_node/http_api/src/validator.rs new file mode 100644 index 00000000000..18e9dbf636b --- /dev/null +++ b/beacon_node/http_api/src/validator.rs @@ -0,0 +1,21 @@ +use beacon_chain::{BeaconChain, BeaconChainError, BeaconChainTypes}; +use types::*; + +/// Uses the `chain.validator_pubkey_cache` to resolve a pubkey to a validator +/// index and then ensures that the validator exists in the given `state`. +pub fn pubkey_to_validator_index( + chain: &BeaconChain, + state: &BeaconState, + pubkey: &PublicKeyBytes, +) -> Result, BeaconChainError> { + chain + .validator_index(pubkey)? + .filter(|&index| { + state + .validators() + .get(index) + .map_or(false, |v| v.pubkey == *pubkey) + }) + .map(Result::Ok) + .transpose() +} diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index efdf66747dc..3ae495378e3 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -30,7 +30,6 @@ use state_processing::per_block_processing::get_expected_withdrawals; use state_processing::per_slot_processing; use std::convert::TryInto; use std::sync::Arc; -use tokio::sync::oneshot; use tokio::time::Duration; use tree_hash::TreeHash; use types::application_domain::ApplicationDomain; @@ -70,7 +69,6 @@ struct ApiTester { attester_slashing: AttesterSlashing, proposer_slashing: ProposerSlashing, voluntary_exit: SignedVoluntaryExit, - _server_shutdown: oneshot::Sender<()>, network_rx: NetworkReceivers, local_enr: Enr, external_peer_id: PeerId, @@ -234,11 +232,10 @@ impl ApiTester { let ApiServer { server, listening_socket: _, - shutdown_tx, network_rx, local_enr, external_peer_id, - } = create_api_server_on_port(chain.clone(), log, port).await; + } = create_api_server_on_port(chain.clone(), &harness.runtime, log, port).await; harness.runtime.task_executor.spawn(server, "api_server"); @@ -266,7 +263,6 @@ impl ApiTester { attester_slashing, proposer_slashing, voluntary_exit, - _server_shutdown: shutdown_tx, network_rx, local_enr, external_peer_id, @@ -320,11 +316,10 @@ impl ApiTester { let ApiServer { server, listening_socket, - shutdown_tx, network_rx, local_enr, external_peer_id, - } = create_api_server(chain.clone(), log).await; + } = create_api_server(chain.clone(), &harness.runtime, log).await; harness.runtime.task_executor.spawn(server, "api_server"); @@ -349,7 +344,6 @@ impl ApiTester { attester_slashing, proposer_slashing, voluntary_exit, - _server_shutdown: shutdown_tx, network_rx, local_enr, external_peer_id, diff --git a/beacon_node/network/src/network_beacon_processor/mod.rs b/beacon_node/network/src/network_beacon_processor/mod.rs index db83bfc1645..f8c4e37ffef 100644 --- a/beacon_node/network/src/network_beacon_processor/mod.rs +++ b/beacon_node/network/src/network_beacon_processor/mod.rs @@ -7,9 +7,9 @@ use beacon_chain::{ }; use beacon_chain::{BeaconChainTypes, NotifyExecutionLayer}; use beacon_processor::{ - work_reprocessing_queue::ReprocessQueueMessage, BeaconProcessorSend, DuplicateCache, - GossipAggregatePackage, GossipAttestationPackage, Work, WorkEvent as BeaconWorkEvent, - MAX_SCHEDULED_WORK_QUEUE_LEN, MAX_WORK_EVENT_QUEUE_LEN, + work_reprocessing_queue::ReprocessQueueMessage, BeaconProcessorChannels, BeaconProcessorSend, + DuplicateCache, GossipAggregatePackage, GossipAttestationPackage, Work, + WorkEvent as BeaconWorkEvent, }; use environment::null_logger; use lighthouse_network::{ @@ -545,11 +545,15 @@ impl NetworkBeaconProcessor> { pub fn null_for_testing( network_globals: Arc>, ) -> (Self, mpsc::Receiver>) { - let (beacon_processor_send, beacon_processor_receive) = - mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN); + let BeaconProcessorChannels { + beacon_processor_tx, + beacon_processor_rx, + work_reprocessing_tx, + work_reprocessing_rx: _work_reprocessing_rx, + } = <_>::default(); + let (network_tx, _network_rx) = mpsc::unbounded_channel(); let (sync_tx, _sync_rx) = mpsc::unbounded_channel(); - let (reprocess_tx, _reprocess_rx) = mpsc::channel(MAX_SCHEDULED_WORK_QUEUE_LEN); let log = null_logger().unwrap(); let harness: BeaconChainHarness> = BeaconChainHarness::builder(E::default()) @@ -562,18 +566,18 @@ impl NetworkBeaconProcessor> { let runtime = TestRuntime::default(); let network_beacon_processor = Self { - beacon_processor_send: BeaconProcessorSend(beacon_processor_send), + beacon_processor_send: beacon_processor_tx, duplicate_cache: DuplicateCache::default(), chain: harness.chain, network_tx, sync_tx, - reprocess_tx, + reprocess_tx: work_reprocessing_tx, network_globals, invalid_block_storage: InvalidBlockStorage::Disabled, executor: runtime.task_executor.clone(), log, }; - (network_beacon_processor, beacon_processor_receive) + (network_beacon_processor, beacon_processor_rx) } } diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index dbe93de1ea9..ce5b9141172 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -11,7 +11,7 @@ use crate::{ use beacon_chain::test_utils::{ AttestationStrategy, BeaconChainHarness, BlockStrategy, EphemeralHarnessType, }; -use beacon_chain::{BeaconChain, ChainConfig}; +use beacon_chain::BeaconChain; use beacon_processor::{work_reprocessing_queue::*, *}; use lighthouse_network::{ discv5::enr::{CombinedKey, EnrBuilder}, @@ -68,16 +68,21 @@ struct TestRig { impl Drop for TestRig { fn drop(&mut self) { // Causes the beacon processor to shutdown. - self.beacon_processor_tx = BeaconProcessorSend(mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN).0); + let len = BeaconProcessorConfig::default().max_work_event_queue_len; + self.beacon_processor_tx = BeaconProcessorSend(mpsc::channel(len).0); } } impl TestRig { pub async fn new(chain_length: u64) -> Self { - Self::new_with_chain_config(chain_length, ChainConfig::default()).await + Self::new_parametric( + chain_length, + BeaconProcessorConfig::default().enable_backfill_rate_limiting, + ) + .await } - pub async fn new_with_chain_config(chain_length: u64, chain_config: ChainConfig) -> Self { + pub async fn new_parametric(chain_length: u64, enable_backfill_rate_limiting: bool) -> Self { // This allows for testing voluntary exits without building out a massive chain. let mut spec = E::default_spec(); spec.shard_committee_period = 2; @@ -86,7 +91,7 @@ impl TestRig { .spec(spec) .deterministic_keypairs(VALIDATOR_COUNT) .fresh_ephemeral_store() - .chain_config(chain_config) + .chain_config(<_>::default()) .build(); harness.advance_slot(); @@ -172,8 +177,15 @@ impl TestRig { let log = harness.logger().clone(); - let (beacon_processor_tx, beacon_processor_rx) = mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN); - let beacon_processor_tx = BeaconProcessorSend(beacon_processor_tx); + let mut beacon_processor_config = BeaconProcessorConfig::default(); + beacon_processor_config.enable_backfill_rate_limiting = enable_backfill_rate_limiting; + let BeaconProcessorChannels { + beacon_processor_tx, + beacon_processor_rx, + work_reprocessing_tx, + work_reprocessing_rx, + } = BeaconProcessorChannels::new(&beacon_processor_config); + let (sync_tx, _sync_rx) = mpsc::unbounded_channel(); // Default metadata @@ -196,8 +208,6 @@ impl TestRig { let executor = harness.runtime.task_executor.clone(); - let (work_reprocessing_tx, work_reprocessing_rx) = - mpsc::channel(MAX_SCHEDULED_WORK_QUEUE_LEN); let (work_journal_tx, work_journal_rx) = mpsc::channel(16_364); let duplicate_cache = DuplicateCache::default(); @@ -220,7 +230,7 @@ impl TestRig { executor, max_workers: cmp::max(1, num_cpus::get()), current_workers: 0, - enable_backfill_rate_limiting: harness.chain.config.enable_backfill_rate_limiting, + config: beacon_processor_config, log: log.clone(), } .spawn_manager( @@ -943,11 +953,8 @@ async fn test_backfill_sync_processing() { /// Ensure that backfill batches get processed as fast as they can when rate-limiting is disabled. #[tokio::test] async fn test_backfill_sync_processing_rate_limiting_disabled() { - let chain_config = ChainConfig { - enable_backfill_rate_limiting: false, - ..Default::default() - }; - let mut rig = TestRig::new_with_chain_config(SMALL_CHAIN, chain_config).await; + let enable_backfill_rate_limiting = false; + let mut rig = TestRig::new_parametric(SMALL_CHAIN, enable_backfill_rate_limiting).await; for _ in 0..3 { rig.enqueue_backfill_batch(); diff --git a/beacon_node/network/src/service/tests.rs b/beacon_node/network/src/service/tests.rs index 9943da15a0d..67f62ff90d6 100644 --- a/beacon_node/network/src/service/tests.rs +++ b/beacon_node/network/src/service/tests.rs @@ -4,15 +4,13 @@ mod tests { use crate::persisted_dht::load_dht; use crate::{NetworkConfig, NetworkService}; use beacon_chain::test_utils::BeaconChainHarness; - use beacon_processor::{ - BeaconProcessorSend, MAX_SCHEDULED_WORK_QUEUE_LEN, MAX_WORK_EVENT_QUEUE_LEN, - }; + use beacon_processor::BeaconProcessorChannels; use lighthouse_network::Enr; use slog::{o, Drain, Level, Logger}; use sloggers::{null::NullLoggerBuilder, Build}; use std::str::FromStr; use std::sync::Arc; - use tokio::{runtime::Runtime, sync::mpsc}; + use tokio::runtime::Runtime; use types::MinimalEthSpec; fn get_logger(actual_log: bool) -> Logger { @@ -70,17 +68,20 @@ mod tests { // Create a new network service which implicitly gets dropped at the // end of the block. - let (beacon_processor_send, _beacon_processor_receive) = - mpsc::channel(MAX_WORK_EVENT_QUEUE_LEN); - let (beacon_processor_reprocess_tx, _beacon_processor_reprocess_rx) = - mpsc::channel(MAX_SCHEDULED_WORK_QUEUE_LEN); + let BeaconProcessorChannels { + beacon_processor_tx, + beacon_processor_rx: _beacon_processor_rx, + work_reprocessing_tx, + work_reprocessing_rx: _work_reprocessing_rx, + } = <_>::default(); + let _network_service = NetworkService::start( beacon_chain.clone(), &config, executor, None, - BeaconProcessorSend(beacon_processor_send), - beacon_processor_reprocess_tx, + beacon_processor_tx, + work_reprocessing_tx, ) .await .unwrap(); diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index e46c3d8ca11..0330bd3f7cc 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -382,6 +382,17 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { stalled. This is useful for very small testnets. TESTING ONLY. DO NOT USE ON \ MAINNET.") ) + .arg( + Arg::with_name("http-enable-beacon-processor") + .long("http-enable-beacon-processor") + .value_name("BOOLEAN") + .help("The beacon processor is a scheduler which provides quality-of-service and \ + DoS protection. When set to \"true\", HTTP API requests will be queued and scheduled \ + alongside other tasks. When set to \"false\", HTTP API responses will be executed \ + immediately.") + .takes_value(true) + .default_value("true") + ) /* Prometheus metrics HTTP server related arguments */ .arg( Arg::with_name("metrics") @@ -1141,4 +1152,55 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .takes_value(true) .possible_values(ProgressiveBalancesMode::VARIANTS) ) + .arg( + Arg::with_name("beacon-processor-max-workers") + .long("beacon-processor-max-workers") + .value_name("INTEGER") + .help("Specifies the maximum concurrent tasks for the task scheduler. Increasing \ + this value may increase resource consumption. Reducing the value \ + may result in decreased resource usage and diminished performance. The \ + default value is the number of logical CPU cores on the host.") + .takes_value(true) + ) + .arg( + Arg::with_name("beacon-processor-work-queue-len") + .long("beacon-processor-work-queue-len") + .value_name("INTEGER") + .help("Specifies the length of the inbound event queue. \ + Higher values may prevent messages from being dropped while lower values \ + may help protect the node from becoming overwhelmed.") + .default_value("16384") + .takes_value(true) + ) + .arg( + Arg::with_name("beacon-processor-reprocess-queue-len") + .long("beacon-processor-reprocess-queue-len") + .value_name("INTEGER") + .help("Specifies the length of the queue for messages requiring delayed processing. \ + Higher values may prevent messages from being dropped while lower values \ + may help protect the node from becoming overwhelmed.") + .default_value("12288") + .takes_value(true) + ) + .arg( + Arg::with_name("beacon-processor-attestation-batch-size") + .long("beacon-processor-attestation-batch-size") + .value_name("INTEGER") + .help("Specifies the number of gossip attestations in a signature verification batch. \ + Higher values may reduce CPU usage in a healthy network whilst lower values may \ + increase CPU usage in an unhealthy or hostile network.") + .default_value("64") + .takes_value(true) + ) + .arg( + Arg::with_name("beacon-processor-aggregate-batch-size") + .long("beacon-processor-aggregate-batch-size") + .value_name("INTEGER") + .help("Specifies the number of gossip aggregate attestations in a signature \ + verification batch. \ + Higher values may reduce CPU usage in a healthy network while lower values may \ + increase CPU usage in an unhealthy or hostile network.") + .default_value("64") + .takes_value(true) + ) } diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index c16b1675a9f..21df86620df 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -4,6 +4,7 @@ use beacon_chain::chain_config::{ }; use clap::ArgMatches; use clap_utils::flags::DISABLE_MALLOC_TUNING_FLAG; +use clap_utils::parse_required; use client::{ClientConfig, ClientGenesis}; use directory::{DEFAULT_BEACON_NODE_DIR, DEFAULT_NETWORK_DIR, DEFAULT_ROOT_DIR}; use environment::RuntimeContext; @@ -148,6 +149,9 @@ pub fn get_config( client_config.http_api.allow_sync_stalled = true; } + client_config.http_api.enable_beacon_processor = + parse_required(cli_args, "http-enable-beacon-processor")?; + if let Some(cache_size) = clap_utils::parse_optional(cli_args, "shuffling-cache-size")? { client_config.chain.shuffling_cache_size = cache_size; } @@ -800,7 +804,7 @@ pub fn get_config( } // Backfill sync rate-limiting - client_config.chain.enable_backfill_rate_limiting = + client_config.beacon_processor.enable_backfill_rate_limiting = !cli_args.is_present("disable-backfill-rate-limiting"); if let Some(path) = clap_utils::parse_optional(cli_args, "invalid-gossip-verified-blocks-path")? @@ -814,6 +818,28 @@ pub fn get_config( client_config.chain.progressive_balances_mode = progressive_balances_mode; } + if let Some(max_workers) = clap_utils::parse_optional(cli_args, "beacon-processor-max-workers")? + { + client_config.beacon_processor.max_workers = max_workers; + } + + if client_config.beacon_processor.max_workers == 0 { + return Err("--beacon-processor-max-workers must be a non-zero value".to_string()); + } + + client_config.beacon_processor.max_work_event_queue_len = + clap_utils::parse_required(cli_args, "beacon-processor-work-queue-len")?; + client_config.beacon_processor.max_scheduled_work_queue_len = + clap_utils::parse_required(cli_args, "beacon-processor-reprocess-queue-len")?; + client_config + .beacon_processor + .max_gossip_attestation_batch_size = + clap_utils::parse_required(cli_args, "beacon-processor-attestation-batch-size")?; + client_config + .beacon_processor + .max_gossip_aggregate_batch_size = + clap_utils::parse_required(cli_args, "beacon-processor-aggregate-batch-size")?; + Ok(client_config) } diff --git a/beacon_node/src/lib.rs b/beacon_node/src/lib.rs index 47694825ca7..3bef69ce83e 100644 --- a/beacon_node/src/lib.rs +++ b/beacon_node/src/lib.rs @@ -83,6 +83,7 @@ impl ProductionBeaconNode { let builder = ClientBuilder::new(context.eth_spec_instance.clone()) .runtime_context(context) .chain_spec(spec) + .beacon_processor(client_config.beacon_processor.clone()) .http_api_config(client_config.http_api.clone()) .disk_store(&db_path, &freezer_db_path, store_config, log.clone())?; diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index 169aa67fdde..8003236f2df 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -66,6 +66,7 @@ lighthouse_network = { path = "../beacon_node/lighthouse_network" } sensitive_url = { path = "../common/sensitive_url" } eth1 = { path = "../beacon_node/eth1" } eth2 = { path = "../common/eth2" } +beacon_processor = { path = "../beacon_node/beacon_processor" } [[test]] name = "lighthouse_tests" diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index bc5f610881b..ecc936cbfb4 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -5,6 +5,7 @@ use beacon_node::beacon_chain::chain_config::{ DisallowedReOrgOffsets, DEFAULT_RE_ORG_CUTOFF_DENOMINATOR, DEFAULT_RE_ORG_MAX_EPOCHS_SINCE_FINALIZATION, DEFAULT_RE_ORG_THRESHOLD, }; +use beacon_processor::BeaconProcessorConfig; use eth1::Eth1Endpoint; use lighthouse_network::PeerId; use std::fs::File; @@ -1118,13 +1119,13 @@ fn disable_backfill_rate_limiting_flag() { CommandLineTest::new() .flag("disable-backfill-rate-limiting", None) .run_with_zero_port() - .with_config(|config| assert!(!config.chain.enable_backfill_rate_limiting)); + .with_config(|config| assert!(!config.beacon_processor.enable_backfill_rate_limiting)); } #[test] fn default_backfill_rate_limiting_flag() { CommandLineTest::new() .run_with_zero_port() - .with_config(|config| assert!(config.chain.enable_backfill_rate_limiting)); + .with_config(|config| assert!(config.beacon_processor.enable_backfill_rate_limiting)); } #[test] fn default_boot_nodes() { @@ -1463,6 +1464,22 @@ fn http_allow_sync_stalled_flag() { .with_config(|config| assert_eq!(config.http_api.allow_sync_stalled, true)); } #[test] +fn http_enable_beacon_processor() { + CommandLineTest::new() + .run_with_zero_port() + .with_config(|config| assert_eq!(config.http_api.enable_beacon_processor, true)); + + CommandLineTest::new() + .flag("http-enable-beacon-processor", Some("true")) + .run_with_zero_port() + .with_config(|config| assert_eq!(config.http_api.enable_beacon_processor, true)); + + CommandLineTest::new() + .flag("http-enable-beacon-processor", Some("false")) + .run_with_zero_port() + .with_config(|config| assert_eq!(config.http_api.enable_beacon_processor, false)); +} +#[test] fn http_tls_flags() { let dir = TempDir::new().expect("Unable to create temporary directory"); CommandLineTest::new() @@ -2295,3 +2312,40 @@ fn progressive_balances_fast() { ) }); } + +#[test] +fn beacon_processor() { + CommandLineTest::new() + .run_with_zero_port() + .with_config(|config| assert_eq!(config.beacon_processor, <_>::default())); + + CommandLineTest::new() + .flag("beacon-processor-max-workers", Some("1")) + .flag("beacon-processor-work-queue-len", Some("2")) + .flag("beacon-processor-reprocess-queue-len", Some("3")) + .flag("beacon-processor-attestation-batch-size", Some("4")) + .flag("beacon-processor-aggregate-batch-size", Some("5")) + .flag("disable-backfill-rate-limiting", None) + .run_with_zero_port() + .with_config(|config| { + assert_eq!( + config.beacon_processor, + BeaconProcessorConfig { + max_workers: 1, + max_work_event_queue_len: 2, + max_scheduled_work_queue_len: 3, + max_gossip_attestation_batch_size: 4, + max_gossip_aggregate_batch_size: 5, + enable_backfill_rate_limiting: false + } + ) + }); +} + +#[test] +#[should_panic] +fn beacon_processor_zero_workers() { + CommandLineTest::new() + .flag("beacon-processor-max-workers", Some("0")) + .run_with_zero_port(); +} diff --git a/testing/node_test_rig/src/lib.rs b/testing/node_test_rig/src/lib.rs index 62db67b8c57..394f8558fae 100644 --- a/testing/node_test_rig/src/lib.rs +++ b/testing/node_test_rig/src/lib.rs @@ -24,7 +24,7 @@ pub use execution_layer::test_utils::{ pub use validator_client::Config as ValidatorConfig; /// The global timeout for HTTP requests to the beacon node. -const HTTP_TIMEOUT: Duration = Duration::from_secs(4); +const HTTP_TIMEOUT: Duration = Duration::from_secs(8); /// The timeout for a beacon node to start up. const STARTUP_TIMEOUT: Duration = Duration::from_secs(60); @@ -115,6 +115,11 @@ pub fn testing_client_config() -> ClientConfig { genesis_time: now, }; + // Specify a constant count of beacon processor workers. Having this number + // too low can cause annoying HTTP timeouts, especially on Github runners + // with 2 logical CPUs. + client_config.beacon_processor.max_workers = 4; + client_config } diff --git a/watch/Cargo.toml b/watch/Cargo.toml index 1a003167dc8..23e2c566dc1 100644 --- a/watch/Cargo.toml +++ b/watch/Cargo.toml @@ -43,3 +43,4 @@ beacon_chain = { path = "../beacon_node/beacon_chain" } network = { path = "../beacon_node/network" } testcontainers = "0.14.0" unused_port = { path = "../common/unused_port" } +task_executor = { path = "../common/task_executor" } diff --git a/watch/tests/tests.rs b/watch/tests/tests.rs index 28700ccdce7..af1cde26b7a 100644 --- a/watch/tests/tests.rs +++ b/watch/tests/tests.rs @@ -85,7 +85,6 @@ struct TesterBuilder { pub harness: BeaconChainHarness>, pub config: Config, _bn_network_rx: NetworkReceivers, - _bn_api_shutdown_tx: oneshot::Sender<()>, } impl TesterBuilder { @@ -102,10 +101,14 @@ impl TesterBuilder { let ApiServer { server, listening_socket: bn_api_listening_socket, - shutdown_tx: _bn_api_shutdown_tx, network_rx: _bn_network_rx, .. - } = create_api_server(harness.chain.clone(), harness.logger().clone()).await; + } = create_api_server( + harness.chain.clone(), + &harness.runtime, + harness.logger().clone(), + ) + .await; tokio::spawn(server); /* @@ -139,7 +142,6 @@ impl TesterBuilder { harness, config, _bn_network_rx, - _bn_api_shutdown_tx, } } pub async fn build(self, pool: PgPool) -> Tester { @@ -186,7 +188,6 @@ impl TesterBuilder { config: self.config, updater, _bn_network_rx: self._bn_network_rx, - _bn_api_shutdown_tx: self._bn_api_shutdown_tx, _watch_shutdown_tx, } } @@ -204,7 +205,6 @@ struct Tester { pub config: Config, pub updater: UpdateHandler, _bn_network_rx: NetworkReceivers, - _bn_api_shutdown_tx: oneshot::Sender<()>, _watch_shutdown_tx: oneshot::Sender<()>, } From c3ced280951a19781bb7620da1c6530c712c0f00 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 9 Aug 2023 10:45:21 -0400 Subject: [PATCH 2/5] cargo fmt --- beacon_node/http_api/src/lib.rs | 75 ++++++++---------------- beacon_node/network/src/service/tests.rs | 3 +- 2 files changed, 27 insertions(+), 51 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 7e0fd0dce9d..4b95a04a4ee 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -1291,8 +1291,8 @@ pub fn serve( log, BroadcastValidation::default(), ) - .await - .map(|()| warp::reply().into_response()) + .await + .map(|()| warp::reply().into_response()) }) }, ); @@ -1348,7 +1348,7 @@ pub fn serve( task_spawner: TaskSpawner, chain: Arc>, network_tx: UnboundedSender>, - log: Logger| { + log: Logger| { task_spawner.spawn_async(Priority::P1, async move { match publish_blocks::publish_block( None, @@ -1358,7 +1358,7 @@ pub fn serve( log, validation_level.broadcast_validation, ) - .await + .await { Ok(()) => warp::reply().into_response(), Err(e) => match warp_utils::reject::handle_rejection(e).await { @@ -1367,7 +1367,7 @@ pub fn serve( StatusCode::INTERNAL_SERVER_ERROR, eth2::StatusCode::INTERNAL_SERVER_ERROR, ) - .into_response(), + .into_response(), }, } }) @@ -1511,27 +1511,27 @@ pub fn serve( task_spawner: TaskSpawner, chain: Arc>, network_tx: UnboundedSender>, - log: Logger| { + log: Logger| { task_spawner.spawn_async(Priority::P0, async move { - match publish_blocks::publish_blinded_block( - block_contents, - chain, - &network_tx, - log, - validation_level.broadcast_validation, - ) - .await - { - Ok(()) => warp::reply().into_response(), - Err(e) => match warp_utils::reject::handle_rejection(e).await { - Ok(reply) => reply.into_response(), - Err(_) => warp::reply::with_status( - StatusCode::INTERNAL_SERVER_ERROR, - eth2::StatusCode::INTERNAL_SERVER_ERROR, - ) - .into_response(), - }, - } + match publish_blocks::publish_blinded_block( + block_contents, + chain, + &network_tx, + log, + validation_level.broadcast_validation, + ) + .await + { + Ok(()) => warp::reply().into_response(), + Err(e) => match warp_utils::reject::handle_rejection(e).await { + Ok(reply) => reply.into_response(), + Err(_) => warp::reply::with_status( + StatusCode::INTERNAL_SERVER_ERROR, + eth2::StatusCode::INTERNAL_SERVER_ERROR, + ) + .into_response(), + }, + } }) }, ); @@ -4251,31 +4251,6 @@ pub fn serve( }, ); - // POST lighthouse/database/historical_blocks - let post_lighthouse_database_historical_blocks = database_path - .and(warp::path("historical_blocks")) - .and(warp::path::end()) - .and(warp::body::json()) - .and(task_spawner_filter.clone()) - .and(chain_filter.clone()) - .and(log_filter.clone()) - .and_then( - |blocks: Vec>>, - task_spawner: TaskSpawner, - chain: Arc>, - log: Logger| { - info!( - log, - "Importing historical blocks"; - "count" => blocks.len(), - "source" => "http_api" - ); - task_spawner.blocking_json_task(Priority::P1, move || { - database::historical_blocks(chain, blocks) - }) - }, - ); - // GET lighthouse/analysis/block_rewards let get_lighthouse_block_rewards = warp::path("lighthouse") .and(warp::path("analysis")) diff --git a/beacon_node/network/src/service/tests.rs b/beacon_node/network/src/service/tests.rs index 66804138448..c9aa592d6a1 100644 --- a/beacon_node/network/src/service/tests.rs +++ b/beacon_node/network/src/service/tests.rs @@ -5,7 +5,8 @@ mod tests { use crate::{NetworkConfig, NetworkService}; use beacon_chain::test_utils::BeaconChainHarness; use beacon_processor::{ - BeaconProcessorChannels, BeaconProcessorSend, MAX_SCHEDULED_WORK_QUEUE_LEN, MAX_WORK_EVENT_QUEUE_LEN, + BeaconProcessorChannels, BeaconProcessorSend, MAX_SCHEDULED_WORK_QUEUE_LEN, + MAX_WORK_EVENT_QUEUE_LEN, }; use lighthouse_network::Enr; use slog::{o, Drain, Level, Logger}; From 4da6ca73d701fbe9aea88479b2e71a1fbf073412 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 9 Aug 2023 14:12:45 -0400 Subject: [PATCH 3/5] fix imports --- beacon_node/network/src/network_beacon_processor/tests.rs | 3 ++- beacon_node/network/src/service/tests.rs | 7 ++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index 11e71d5967e..d1e38d6513f 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -30,7 +30,7 @@ use std::time::Duration; use tokio::sync::mpsc; use types::blob_sidecar::FixedBlobSidecarList; use types::{ - Attestation, AttesterSlashing, Epoch, Hash256, MainnetEthSpec, ProposerSlashing, + Attestation, AttesterSlashing, Epoch, EthSpec, Hash256, MainnetEthSpec, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBlobSidecarList, SignedVoluntaryExit, Slot, SubnetId, }; @@ -90,6 +90,7 @@ impl TestRig { pub async fn new_parametric(chain_length: u64, enable_backfill_rate_limiting: bool) -> Self { // This allows for testing voluntary exits without building out a massive chain. + let mut spec = E::default_spec(); spec.shard_committee_period = 2; let harness = BeaconChainHarness::builder(MainnetEthSpec) diff --git a/beacon_node/network/src/service/tests.rs b/beacon_node/network/src/service/tests.rs index c9aa592d6a1..64ac3aaeeee 100644 --- a/beacon_node/network/src/service/tests.rs +++ b/beacon_node/network/src/service/tests.rs @@ -4,16 +4,13 @@ mod tests { use crate::persisted_dht::load_dht; use crate::{NetworkConfig, NetworkService}; use beacon_chain::test_utils::BeaconChainHarness; - use beacon_processor::{ - BeaconProcessorChannels, BeaconProcessorSend, MAX_SCHEDULED_WORK_QUEUE_LEN, - MAX_WORK_EVENT_QUEUE_LEN, - }; + use beacon_processor::BeaconProcessorChannels; use lighthouse_network::Enr; use slog::{o, Drain, Level, Logger}; use sloggers::{null::NullLoggerBuilder, Build}; use std::str::FromStr; use std::sync::Arc; - use tokio::{runtime::Runtime, sync::mpsc}; + use tokio::runtime::Runtime; use types::MinimalEthSpec as E; fn get_logger(actual_log: bool) -> Logger { From 9e8a289d21f2708d34a4375c7a3390e0318e94e4 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Wed, 9 Aug 2023 15:36:39 -0400 Subject: [PATCH 4/5] fix blobs by range test --- beacon_node/network/src/network_beacon_processor/tests.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index d1e38d6513f..57728775034 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -30,7 +30,7 @@ use std::time::Duration; use tokio::sync::mpsc; use types::blob_sidecar::FixedBlobSidecarList; use types::{ - Attestation, AttesterSlashing, Epoch, EthSpec, Hash256, MainnetEthSpec, ProposerSlashing, + Attestation, AttesterSlashing, Epoch, Hash256, MainnetEthSpec, ProposerSlashing, SignedAggregateAndProof, SignedBeaconBlock, SignedBlobSidecarList, SignedVoluntaryExit, Slot, SubnetId, }; @@ -90,7 +90,7 @@ impl TestRig { pub async fn new_parametric(chain_length: u64, enable_backfill_rate_limiting: bool) -> Self { // This allows for testing voluntary exits without building out a massive chain. - let mut spec = E::default_spec(); + let mut spec = test_spec::(); spec.shard_committee_period = 2; let harness = BeaconChainHarness::builder(MainnetEthSpec) From 2b5385fb461293da325858c389c0a31c3bee98b3 Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Wed, 9 Aug 2023 14:44:47 -0500 Subject: [PATCH 5/5] Changes for `devnet-8` (#4518) * Addressed #4487 Add override threshold flag Added tests for Override Threshold Flag Override default shown in decimal * Addressed #4445 Addressed Jimmy's Comments No need for matches Fix Mock Execution Engine Tests Fix clippy fix fcuv3 bug * Fix Block Root Calculation post-Deneb * Addressed #4444 Attestation Verification Post-Deneb Fix Gossip Attestation Verification Test * Addressed #4443 Fix Exit Signing for EIP-7044 Fix cross exit test Move 7044 Logic to signing_context() * Update EF Tests * Addressed #4560 * Added Comments around EIP7045 * Combine Altair Deneb to Eliminate Duplicated Code --- account_manager/src/validator/exit.rs | 21 +- .../src/attestation_verification.rs | 15 +- .../beacon_chain/src/beacon_block_reward.rs | 19 +- beacon_node/beacon_chain/src/beacon_chain.rs | 11 +- .../beacon_chain/src/execution_payload.rs | 49 ++--- beacon_node/beacon_chain/src/test_utils.rs | 3 +- .../tests/attestation_verification.rs | 48 +++-- .../tests/payload_invalidation.rs | 1 + beacon_node/execution_layer/src/block_hash.rs | 53 +++-- beacon_node/execution_layer/src/engine_api.rs | 191 ++++++++++++++++-- .../execution_layer/src/engine_api/http.rs | 90 +++++++-- .../src/engine_api/json_structures.rs | 35 +++- beacon_node/execution_layer/src/lib.rs | 162 +++++++++++---- .../test_utils/execution_block_generator.rs | 38 ++-- .../src/test_utils/handle_rpc.rs | 37 +++- .../src/test_utils/mock_builder.rs | 37 +++- .../src/test_utils/mock_execution_layer.rs | 16 +- .../execution_layer/src/test_utils/mod.rs | 1 + beacon_node/operation_pool/src/attestation.rs | 4 +- beacon_node/operation_pool/src/lib.rs | 16 +- beacon_node/src/cli.rs | 18 ++ beacon_node/src/config.rs | 2 + common/eth2/src/types.rs | 6 +- .../common/get_attestation_participation.rs | 17 +- .../process_operations.rs | 4 +- .../per_block_processing/signature_sets.rs | 23 ++- .../verify_attestation.rs | 21 +- consensus/types/src/execution_block_header.rs | 20 +- consensus/types/src/execution_payload.rs | 10 +- .../types/src/execution_payload_header.rs | 12 +- consensus/types/src/voluntary_exit.rs | 20 +- lighthouse/tests/beacon_node.rs | 32 +++ testing/ef_tests/Makefile | 2 +- testing/ef_tests/src/cases/operations.rs | 11 +- .../src/test_rig.rs | 31 ++- validator_client/src/validator_store.rs | 46 ++++- 36 files changed, 842 insertions(+), 280 deletions(-) diff --git a/account_manager/src/validator/exit.rs b/account_manager/src/validator/exit.rs index 5755a355f31..805c61ef535 100644 --- a/account_manager/src/validator/exit.rs +++ b/account_manager/src/validator/exit.rs @@ -14,7 +14,7 @@ use slot_clock::{SlotClock, SystemTimeSlotClock}; use std::path::{Path, PathBuf}; use std::time::Duration; use tokio::time::sleep; -use types::{ChainSpec, Epoch, EthSpec, Fork, VoluntaryExit}; +use types::{ChainSpec, Epoch, EthSpec, VoluntaryExit}; pub const CMD: &str = "exit"; pub const KEYSTORE_FLAG: &str = "keystore"; @@ -148,7 +148,6 @@ async fn publish_voluntary_exit( .ok_or("Failed to get current epoch. Please check your system time")?; let validator_index = get_validator_index_for_exit(client, &keypair.pk, epoch, spec).await?; - let fork = get_beacon_state_fork(client).await?; let voluntary_exit = VoluntaryExit { epoch, validator_index, @@ -175,12 +174,8 @@ async fn publish_voluntary_exit( if confirmation == CONFIRMATION_PHRASE { // Sign and publish the voluntary exit to network - let signed_voluntary_exit = voluntary_exit.sign( - &keypair.sk, - &fork, - genesis_data.genesis_validators_root, - spec, - ); + let signed_voluntary_exit = + voluntary_exit.sign(&keypair.sk, genesis_data.genesis_validators_root, spec); client .post_beacon_pool_voluntary_exits(&signed_voluntary_exit) .await @@ -318,16 +313,6 @@ async fn is_syncing(client: &BeaconNodeHttpClient) -> Result { .is_syncing) } -/// Get fork object for the current state by querying the beacon node client. -async fn get_beacon_state_fork(client: &BeaconNodeHttpClient) -> Result { - Ok(client - .get_beacon_states_fork(StateId::Head) - .await - .map_err(|e| format!("Failed to get get fork: {:?}", e))? - .ok_or("Failed to get fork, state not found")? - .data) -} - /// Calculates the current epoch from the genesis time and current time. fn get_current_epoch(genesis_time: u64, spec: &ChainSpec) -> Option { let slot_clock = SystemTimeSlotClock::new( diff --git a/beacon_node/beacon_chain/src/attestation_verification.rs b/beacon_node/beacon_chain/src/attestation_verification.rs index 5535fec37c4..d7a8bca4d0f 100644 --- a/beacon_node/beacon_chain/src/attestation_verification.rs +++ b/beacon_node/beacon_chain/src/attestation_verification.rs @@ -55,7 +55,7 @@ use std::borrow::Cow; use strum::AsRefStr; use tree_hash::TreeHash; use types::{ - Attestation, BeaconCommittee, ChainSpec, CommitteeIndex, Epoch, EthSpec, Hash256, + Attestation, BeaconCommittee, ChainSpec, CommitteeIndex, Epoch, EthSpec, ForkName, Hash256, IndexedAttestation, SelectionProof, SignedAggregateAndProof, Slot, SubnetId, }; @@ -1049,10 +1049,21 @@ pub fn verify_propagation_slot_range( } // Taking advantage of saturating subtraction on `Slot`. - let earliest_permissible_slot = slot_clock + let one_epoch_prior = slot_clock .now_with_past_tolerance(spec.maximum_gossip_clock_disparity()) .ok_or(BeaconChainError::UnableToReadSlot)? - E::slots_per_epoch(); + + let current_fork = + spec.fork_name_at_slot::(slot_clock.now().ok_or(BeaconChainError::UnableToReadSlot)?); + let earliest_permissible_slot = match current_fork { + ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => one_epoch_prior, + // EIP-7045 + ForkName::Deneb => one_epoch_prior + .epoch(E::slots_per_epoch()) + .start_slot(E::slots_per_epoch()), + }; + if attestation_slot < earliest_permissible_slot { return Err(Error::PastSlot { attestation_slot, diff --git a/beacon_node/beacon_chain/src/beacon_block_reward.rs b/beacon_node/beacon_chain/src/beacon_block_reward.rs index 786402c9978..8cbeae371ec 100644 --- a/beacon_node/beacon_chain/src/beacon_block_reward.rs +++ b/beacon_node/beacon_chain/src/beacon_block_reward.rs @@ -64,19 +64,19 @@ impl BeaconChain { self.compute_beacon_block_attestation_reward_base(block, block_root, state) .map_err(|e| { error!( - self.log, - "Error calculating base block attestation reward"; - "error" => ?e + self.log, + "Error calculating base block attestation reward"; + "error" => ?e ); BeaconChainError::BlockRewardAttestationError })? } else { - self.compute_beacon_block_attestation_reward_altair(block, state) + self.compute_beacon_block_attestation_reward_altair_deneb(block, state) .map_err(|e| { error!( - self.log, - "Error calculating altair block attestation reward"; - "error" => ?e + self.log, + "Error calculating altair block attestation reward"; + "error" => ?e ); BeaconChainError::BlockRewardAttestationError })? @@ -173,7 +173,9 @@ impl BeaconChain { Ok(block_attestation_reward) } - fn compute_beacon_block_attestation_reward_altair>( + fn compute_beacon_block_attestation_reward_altair_deneb< + Payload: AbstractExecPayload, + >( &self, block: BeaconBlockRef<'_, T::EthSpec, Payload>, state: &mut BeaconState, @@ -192,6 +194,7 @@ impl BeaconChain { for attestation in block.body().attestations() { let data = &attestation.data; let inclusion_delay = state.slot().safe_sub(data.slot)?.as_u64(); + // [Modified in Deneb:EIP7045] let participation_flag_indices = get_attestation_participation_flag_indices( state, data, diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 3489fb7ab64..0a89b739103 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -237,6 +237,7 @@ pub struct PrePayloadAttributes { /// The parent block number is not part of the payload attributes sent to the EL, but *is* /// sent to builders via SSE. pub parent_block_number: u64, + pub parent_beacon_block_root: Hash256, } /// Information about a state/block at a specific slot. @@ -4200,6 +4201,7 @@ impl BeaconChain { proposer_index, prev_randao, parent_block_number, + parent_beacon_block_root: parent_block_root, })) } @@ -5260,7 +5262,8 @@ impl BeaconChain { { payload_attributes } else { - let withdrawals = match self.spec.fork_name_at_slot::(prepare_slot) { + let prepare_slot_fork = self.spec.fork_name_at_slot::(prepare_slot); + let withdrawals = match prepare_slot_fork { ForkName::Base | ForkName::Altair | ForkName::Merge => None, ForkName::Capella | ForkName::Deneb => { let chain = self.clone(); @@ -5275,6 +5278,11 @@ impl BeaconChain { } }; + let parent_beacon_block_root = match prepare_slot_fork { + ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => None, + ForkName::Deneb => Some(pre_payload_attributes.parent_beacon_block_root), + }; + let payload_attributes = PayloadAttributes::new( self.slot_clock .start_of(prepare_slot) @@ -5283,6 +5291,7 @@ impl BeaconChain { pre_payload_attributes.prev_randao, execution_layer.get_suggested_fee_recipient(proposer).await, withdrawals.map(Into::into), + parent_beacon_block_root, ); execution_layer diff --git a/beacon_node/beacon_chain/src/execution_payload.rs b/beacon_node/beacon_chain/src/execution_payload.rs index 1166de1e563..7a095fce4de 100644 --- a/beacon_node/beacon_chain/src/execution_payload.rs +++ b/beacon_node/beacon_chain/src/execution_payload.rs @@ -12,13 +12,15 @@ use crate::{ BeaconChain, BeaconChainError, BeaconChainTypes, BlockError, BlockProductionError, ExecutionPayloadError, }; -use execution_layer::{BlockProposalContents, BuilderParams, PayloadAttributes, PayloadStatus}; +use execution_layer::{ + BlockProposalContents, BuilderParams, NewPayloadRequest, PayloadAttributes, PayloadStatus, +}; use fork_choice::{InvalidationOperation, PayloadVerificationStatus}; use proto_array::{Block as ProtoBlock, ExecutionStatus}; use slog::{debug, warn}; use slot_clock::SlotClock; use state_processing::per_block_processing::{ - self, compute_timestamp_at_slot, get_expected_withdrawals, is_execution_enabled, + compute_timestamp_at_slot, get_expected_withdrawals, is_execution_enabled, is_merge_transition_complete, partially_verify_execution_payload, }; use std::sync::Arc; @@ -76,8 +78,6 @@ impl PayloadNotifier { ) .map_err(BlockError::PerBlockProcessingError)?; - let payload = block_message.execution_payload()?; - match notify_execution_layer { NotifyExecutionLayer::No if chain.config.optimistic_finalized_sync => { // Verify the block hash here in Lighthouse and immediately mark the block as @@ -87,13 +87,11 @@ impl PayloadNotifier { .as_ref() .ok_or(ExecutionPayloadError::NoExecutionConnection)?; - if let Err(e) = - execution_layer.verify_payload_block_hash(payload.execution_payload_ref()) - { + if let Err(e) = execution_layer.verify_payload_block_hash(block_message) { warn!( chain.log, "Falling back to slow block hash verification"; - "block_number" => payload.block_number(), + "block_number" => ?block_message.execution_payload().map(|payload| payload.block_number()), "info" => "you can silence this warning with --disable-optimistic-finalized-sync", "error" => ?e, ); @@ -139,23 +137,15 @@ async fn notify_new_payload<'a, T: BeaconChainTypes>( chain: &Arc>, block: BeaconBlockRef<'a, T::EthSpec>, ) -> Result> { - let execution_payload = block.execution_payload()?; - let versioned_hashes = block.body().blob_kzg_commitments().ok().map(|commitments| { - commitments - .into_iter() - .map(|commitment| { - per_block_processing::deneb::deneb::kzg_commitment_to_versioned_hash(commitment) - }) - .collect::>() - }); - let execution_layer = chain .execution_layer .as_ref() .ok_or(ExecutionPayloadError::NoExecutionConnection)?; + let new_payload_request: NewPayloadRequest = block.try_into()?; + let execution_block_hash = new_payload_request.block_hash(); let new_payload_response = execution_layer - .notify_new_payload(&execution_payload.into(), versioned_hashes) + .notify_new_payload(new_payload_request) .await; match new_payload_response { @@ -173,7 +163,7 @@ async fn notify_new_payload<'a, T: BeaconChainTypes>( "Invalid execution payload"; "validation_error" => ?validation_error, "latest_valid_hash" => ?latest_valid_hash, - "execution_block_hash" => ?execution_payload.block_hash(), + "execution_block_hash" => ?execution_block_hash, "root" => ?block.tree_hash_root(), "graffiti" => block.body().graffiti().as_utf8_lossy(), "proposer_index" => block.proposer_index(), @@ -219,7 +209,7 @@ async fn notify_new_payload<'a, T: BeaconChainTypes>( chain.log, "Invalid execution payload block hash"; "validation_error" => ?validation_error, - "execution_block_hash" => ?execution_payload.block_hash(), + "execution_block_hash" => ?execution_block_hash, "root" => ?block.tree_hash_root(), "graffiti" => block.body().graffiti().as_utf8_lossy(), "proposer_index" => block.proposer_index(), @@ -435,6 +425,12 @@ pub fn get_execution_payload< // These shouldn't happen but they're here to make the pattern irrefutable &BeaconState::Base(_) | &BeaconState::Altair(_) => None, }; + let parent_beacon_block_root = match state { + &BeaconState::Deneb(_) => Some(state.latest_block_header().canonical_root()), + &BeaconState::Merge(_) | &BeaconState::Capella(_) => None, + // These shouldn't happen but they're here to make the pattern irrefutable + &BeaconState::Base(_) | &BeaconState::Altair(_) => None, + }; // Spawn a task to obtain the execution payload from the EL via a series of async calls. The // `join_handle` can be used to await the result of the function. @@ -452,6 +448,7 @@ pub fn get_execution_payload< latest_execution_payload_header_block_hash, builder_params, withdrawals, + parent_beacon_block_root, ) .await }, @@ -486,6 +483,7 @@ pub async fn prepare_execution_payload( latest_execution_payload_header_block_hash: ExecutionBlockHash, builder_params: BuilderParams, withdrawals: Option>, + parent_beacon_block_root: Option, ) -> Result, BlockProductionError> where T: BeaconChainTypes, @@ -547,8 +545,13 @@ where let suggested_fee_recipient = execution_layer .get_suggested_fee_recipient(proposer_index) .await; - let payload_attributes = - PayloadAttributes::new(timestamp, random, suggested_fee_recipient, withdrawals); + let payload_attributes = PayloadAttributes::new( + timestamp, + random, + suggested_fee_recipient, + withdrawals, + parent_beacon_block_root, + ); // Note: the suggested_fee_recipient is stored in the `execution_layer`, it will add this parameter. // diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index 06fad133015..b8e32ef7fbe 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1663,14 +1663,13 @@ where pub fn make_voluntary_exit(&self, validator_index: u64, epoch: Epoch) -> SignedVoluntaryExit { let sk = &self.validator_keypairs[validator_index as usize].sk; - let fork = self.chain.canonical_head.cached_head().head_fork(); let genesis_validators_root = self.chain.genesis_validators_root; VoluntaryExit { epoch, validator_index, } - .sign(sk, &fork, genesis_validators_root, &self.chain.spec) + .sign(sk, genesis_validators_root, &self.chain.spec) } pub fn add_proposer_slashing(&self, validator_index: u64) -> Result<(), String> { diff --git a/beacon_node/beacon_chain/tests/attestation_verification.rs b/beacon_node/beacon_chain/tests/attestation_verification.rs index 95abef0e6e7..f6a0a5b95ab 100644 --- a/beacon_node/beacon_chain/tests/attestation_verification.rs +++ b/beacon_node/beacon_chain/tests/attestation_verification.rs @@ -326,10 +326,28 @@ impl GossipTester { self.harness.chain.epoch().unwrap() } - pub fn two_epochs_ago(&self) -> Slot { + pub fn earliest_valid_attestation_slot(&self) -> Slot { + let offset = match self.harness.spec.fork_name_at_epoch(self.epoch()) { + ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + // Subtract an additional slot since the harness will be exactly on the start of the + // slot and the propagation tolerance will allow an extra slot. + E::slots_per_epoch() + 1 + } + // EIP-7045 + ForkName::Deneb => { + let epoch_slot_offset = (self.slot() % E::slots_per_epoch()).as_u64(); + if epoch_slot_offset != 0 { + E::slots_per_epoch() + epoch_slot_offset + } else { + // Here the propagation tolerance will cause the cutoff to be an entire epoch earlier + 2 * E::slots_per_epoch() + } + } + }; + self.slot() .as_u64() - .checked_sub(E::slots_per_epoch() + 2) + .checked_sub(offset) .expect("chain is not sufficiently deep for test") .into() } @@ -476,18 +494,21 @@ async fn aggregated_gossip_verification() { ) .inspect_aggregate_err( "aggregate from past slot", - |tester, a| a.message.aggregate.data.slot = tester.two_epochs_ago(), + |tester, a| { + let too_early_slot = tester.earliest_valid_attestation_slot() - 1; + a.message.aggregate.data.slot = too_early_slot; + a.message.aggregate.data.target.epoch = too_early_slot.epoch(E::slots_per_epoch()); + }, |tester, err| { + let valid_early_slot = tester.earliest_valid_attestation_slot(); assert!(matches!( err, AttnError::PastSlot { attestation_slot, - // Subtract an additional slot since the harness will be exactly on the start of the - // slot and the propagation tolerance will allow an extra slot. earliest_permissible_slot } - if attestation_slot == tester.two_epochs_ago() - && earliest_permissible_slot == tester.slot() - E::slots_per_epoch() - 1 + if attestation_slot == valid_early_slot - 1 + && earliest_permissible_slot == valid_early_slot )) }, ) @@ -792,21 +813,20 @@ async fn unaggregated_gossip_verification() { .inspect_unaggregate_err( "attestation from past slot", |tester, a, _| { - let early_slot = tester.two_epochs_ago(); - a.data.slot = early_slot; - a.data.target.epoch = early_slot.epoch(E::slots_per_epoch()); + let too_early_slot = tester.earliest_valid_attestation_slot() - 1; + a.data.slot = too_early_slot; + a.data.target.epoch = too_early_slot.epoch(E::slots_per_epoch()); }, |tester, err| { + let valid_early_slot = tester.earliest_valid_attestation_slot(); assert!(matches!( err, AttnError::PastSlot { attestation_slot, - // Subtract an additional slot since the harness will be exactly on the start of the - // slot and the propagation tolerance will allow an extra slot. earliest_permissible_slot, } - if attestation_slot == tester.two_epochs_ago() - && earliest_permissible_slot == tester.slot() - E::slots_per_epoch() - 1 + if attestation_slot == valid_early_slot - 1 + && earliest_permissible_slot == valid_early_slot )) }, ) diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index fb7cc516ff3..399487928e3 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -1017,6 +1017,7 @@ async fn payload_preparation() { .unwrap(), fee_recipient, None, + None, ); assert_eq!(rig.previous_payload_attributes(), payload_attributes); } diff --git a/beacon_node/execution_layer/src/block_hash.rs b/beacon_node/execution_layer/src/block_hash.rs index 73d7643eb34..93931aaa51c 100644 --- a/beacon_node/execution_layer/src/block_hash.rs +++ b/beacon_node/execution_layer/src/block_hash.rs @@ -7,8 +7,8 @@ use ethers_core::utils::rlp::RlpStream; use keccak_hash::KECCAK_EMPTY_LIST_RLP; use triehash::ordered_trie_root; use types::{ - map_execution_block_header_fields_except_withdrawals, Address, EthSpec, ExecutionBlockHash, - ExecutionBlockHeader, ExecutionPayloadRef, Hash256, Hash64, Uint256, + map_execution_block_header_fields_except_withdrawals, Address, BeaconBlockRef, EthSpec, + ExecutionBlockHash, ExecutionBlockHeader, ExecutionPayloadRef, Hash256, Hash64, Uint256, }; impl ExecutionLayer { @@ -18,6 +18,7 @@ impl ExecutionLayer { /// transactions. pub fn calculate_execution_block_hash( payload: ExecutionPayloadRef, + parent_beacon_block_root: Hash256, ) -> (ExecutionBlockHash, Hash256) { // Calculate the transactions root. // We're currently using a deprecated Parity library for this. We should move to a @@ -37,8 +38,13 @@ impl ExecutionLayer { None }; - let rlp_data_gas_used = payload.data_gas_used().ok(); - let rlp_excess_data_gas = payload.excess_data_gas().ok(); + let rlp_blob_gas_used = payload.blob_gas_used().ok(); + let rlp_excess_blob_gas = payload.excess_blob_gas().ok(); + + // Calculate parent beacon block root (post-Deneb). + let rlp_parent_beacon_block_root = rlp_excess_blob_gas + .as_ref() + .map(|_| parent_beacon_block_root); // Construct the block header. let exec_block_header = ExecutionBlockHeader::from_payload( @@ -46,8 +52,9 @@ impl ExecutionLayer { KECCAK_EMPTY_LIST_RLP.as_fixed_bytes().into(), rlp_transactions_root, rlp_withdrawals_root, - rlp_data_gas_used.copied(), - rlp_excess_data_gas.copied(), + rlp_blob_gas_used, + rlp_excess_blob_gas, + rlp_parent_beacon_block_root, ); // Hash the RLP encoding of the block header. @@ -61,10 +68,14 @@ impl ExecutionLayer { /// Verify `payload.block_hash` locally within Lighthouse. /// /// No remote calls to the execution client will be made, so this is quite a cheap check. - pub fn verify_payload_block_hash(&self, payload: ExecutionPayloadRef) -> Result<(), Error> { + pub fn verify_payload_block_hash(&self, block: BeaconBlockRef) -> Result<(), Error> { + let payload = block.execution_payload()?.execution_payload_ref(); + let parent_beacon_block_root = block.parent_root(); + let _timer = metrics::start_timer(&metrics::EXECUTION_LAYER_VERIFY_BLOCK_HASH); - let (header_hash, rlp_transactions_root) = Self::calculate_execution_block_hash(payload); + let (header_hash, rlp_transactions_root) = + Self::calculate_execution_block_hash(payload, parent_beacon_block_root); if header_hash != payload.block_hash() { return Err(Error::BlockHashMismatch { @@ -99,11 +110,14 @@ pub fn rlp_encode_block_header(header: &ExecutionBlockHeader) -> Vec { if let Some(withdrawals_root) = &header.withdrawals_root { rlp_header_stream.append(withdrawals_root); } - if let Some(data_gas_used) = &header.data_gas_used { - rlp_header_stream.append(data_gas_used); + if let Some(blob_gas_used) = &header.blob_gas_used { + rlp_header_stream.append(blob_gas_used); + } + if let Some(excess_blob_gas) = &header.excess_blob_gas { + rlp_header_stream.append(excess_blob_gas); } - if let Some(excess_data_gas) = &header.excess_data_gas { - rlp_header_stream.append(excess_data_gas); + if let Some(parent_beacon_block_root) = &header.parent_beacon_block_root { + rlp_header_stream.append(parent_beacon_block_root); } rlp_header_stream.finalize_unbounded_list(); rlp_header_stream.out().into() @@ -151,8 +165,9 @@ mod test { nonce: Hash64::zero(), base_fee_per_gas: 0x036b_u64.into(), withdrawals_root: None, - data_gas_used: None, - excess_data_gas: None, + blob_gas_used: None, + excess_blob_gas: None, + parent_beacon_block_root: None, }; let expected_rlp = "f90200a0e0a94a7a3c9617401586b1a27025d2d9671332d22d540e0af72b069170380f2aa01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794ba5e000000000000000000000000000000000000a0ec3c94b18b8a1cff7d60f8d258ec723312932928626b4c9355eb4ab3568ec7f7a050f738580ed699f0469702c7ccc63ed2e51bc034be9479b7bff4e68dee84accfa029b0562f7140574dd0d50dee8a271b22e1a0a7b78fca58f7c60370d8317ba2a9b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000830200000188016345785d8a00008301553482079e42a0000000000000000000000000000000000000000000000000000000000000000088000000000000000082036b"; let expected_hash = @@ -181,8 +196,9 @@ mod test { nonce: Hash64::zero(), base_fee_per_gas: 0x036b_u64.into(), withdrawals_root: None, - data_gas_used: None, - excess_data_gas: None, + blob_gas_used: None, + excess_blob_gas: None, + parent_beacon_block_root: None, }; let expected_rlp = "f901fda0927ca537f06c783a3a2635b8805eef1c8c2124f7444ad4a3389898dd832f2dbea01dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d4934794ba5e000000000000000000000000000000000000a0e97859b065bd8dbbb4519c7cb935024de2484c2b7f881181b4360492f0b06b82a050f738580ed699f0469702c7ccc63ed2e51bc034be9479b7bff4e68dee84accfa029b0562f7140574dd0d50dee8a271b22e1a0a7b78fca58f7c60370d8317ba2a9b9010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800188016345785d8a00008301553482079e42a0000000000000000000000000000000000000000000000000000000000002000088000000000000000082036b"; let expected_hash = @@ -212,8 +228,9 @@ mod test { nonce: Hash64::zero(), base_fee_per_gas: 0x34187b238_u64.into(), withdrawals_root: None, - data_gas_used: None, - excess_data_gas: None, + blob_gas_used: None, + excess_blob_gas: None, + parent_beacon_block_root: None, }; let expected_hash = Hash256::from_str("6da69709cd5a34079b6604d29cd78fc01dacd7c6268980057ad92a2bede87351") diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 9dce3c0477c..ed3cc330a39 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -1,12 +1,14 @@ use crate::engines::ForkchoiceState; use crate::http::{ - ENGINE_FORKCHOICE_UPDATED_V1, ENGINE_FORKCHOICE_UPDATED_V2, + ENGINE_FORKCHOICE_UPDATED_V1, ENGINE_FORKCHOICE_UPDATED_V2, ENGINE_FORKCHOICE_UPDATED_V3, ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1, ENGINE_GET_PAYLOAD_BODIES_BY_RANGE_V1, ENGINE_GET_PAYLOAD_V1, ENGINE_GET_PAYLOAD_V2, ENGINE_GET_PAYLOAD_V3, ENGINE_NEW_PAYLOAD_V1, ENGINE_NEW_PAYLOAD_V2, ENGINE_NEW_PAYLOAD_V3, }; use crate::BlobTxConversionError; -use eth2::types::{SsePayloadAttributes, SsePayloadAttributesV1, SsePayloadAttributesV2}; +use eth2::types::{ + SsePayloadAttributes, SsePayloadAttributesV1, SsePayloadAttributesV2, SsePayloadAttributesV3, +}; use ethers_core::types::Transaction; use ethers_core::utils::rlp::{self, Decodable, Rlp}; use http::deposit_methods::RpcError; @@ -14,17 +16,21 @@ pub use json_structures::{JsonWithdrawal, TransitionConfigurationV1}; use pretty_reqwest_error::PrettyReqwestError; use reqwest::StatusCode; use serde::{Deserialize, Serialize}; +use state_processing::per_block_processing::deneb::deneb::kzg_commitment_to_versioned_hash; use std::convert::TryFrom; use strum::IntoStaticStr; use superstruct::superstruct; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::Blobs; pub use types::{ - Address, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader, + Address, BeaconBlockRef, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader, ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, Uint256, VariableList, Withdrawal, Withdrawals, }; -use types::{ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, KzgProofs}; +use types::{ + BeaconStateError, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, + KzgProofs, VersionedHash, +}; pub mod auth; pub mod http; @@ -191,10 +197,10 @@ pub struct ExecutionBlockWithTransactions { pub withdrawals: Vec, #[superstruct(only(Deneb))] #[serde(with = "serde_utils::u64_hex_be")] - pub data_gas_used: u64, + pub blob_gas_used: u64, #[superstruct(only(Deneb))] #[serde(with = "serde_utils::u64_hex_be")] - pub excess_data_gas: u64, + pub excess_blob_gas: u64, } impl TryFrom> for ExecutionBlockWithTransactions { @@ -271,8 +277,8 @@ impl TryFrom> for ExecutionBlockWithTransactions .into_iter() .map(|withdrawal| withdrawal.into()) .collect(), - data_gas_used: block.data_gas_used, - excess_data_gas: block.excess_data_gas, + blob_gas_used: block.blob_gas_used, + excess_blob_gas: block.excess_blob_gas, }), }; Ok(json_payload) @@ -280,7 +286,7 @@ impl TryFrom> for ExecutionBlockWithTransactions } #[superstruct( - variants(V1, V2), + variants(V1, V2, V3), variant_attributes(derive(Clone, Debug, Eq, Hash, PartialEq),), cast_error(ty = "Error", expr = "Error::IncorrectStateVariant"), partial_getter_error(ty = "Error", expr = "Error::IncorrectStateVariant") @@ -293,8 +299,10 @@ pub struct PayloadAttributes { pub prev_randao: Hash256, #[superstruct(getter(copy))] pub suggested_fee_recipient: Address, - #[superstruct(only(V2))] + #[superstruct(only(V2, V3))] pub withdrawals: Vec, + #[superstruct(only(V3), partial_getter(copy))] + pub parent_beacon_block_root: Hash256, } impl PayloadAttributes { @@ -303,14 +311,24 @@ impl PayloadAttributes { prev_randao: Hash256, suggested_fee_recipient: Address, withdrawals: Option>, + parent_beacon_block_root: Option, ) -> Self { match withdrawals { - Some(withdrawals) => PayloadAttributes::V2(PayloadAttributesV2 { - timestamp, - prev_randao, - suggested_fee_recipient, - withdrawals, - }), + Some(withdrawals) => match parent_beacon_block_root { + Some(parent_beacon_block_root) => PayloadAttributes::V3(PayloadAttributesV3 { + timestamp, + prev_randao, + suggested_fee_recipient, + withdrawals, + parent_beacon_block_root, + }), + None => PayloadAttributes::V2(PayloadAttributesV2 { + timestamp, + prev_randao, + suggested_fee_recipient, + withdrawals, + }), + }, None => PayloadAttributes::V1(PayloadAttributesV1 { timestamp, prev_randao, @@ -343,6 +361,19 @@ impl From for SsePayloadAttributes { suggested_fee_recipient, withdrawals, }), + PayloadAttributes::V3(PayloadAttributesV3 { + timestamp, + prev_randao, + suggested_fee_recipient, + withdrawals, + parent_beacon_block_root, + }) => Self::V3(SsePayloadAttributesV3 { + timestamp, + prev_randao, + suggested_fee_recipient, + withdrawals, + parent_beacon_block_root, + }), } } } @@ -386,6 +417,22 @@ pub struct GetPayloadResponse { pub block_value: Uint256, #[superstruct(only(Deneb))] pub blobs_bundle: BlobsBundleV1, + #[superstruct(only(Deneb), partial_getter(copy))] + pub should_override_builder: bool, +} + +impl GetPayloadResponse { + pub fn fee_recipient(&self) -> Address { + ExecutionPayloadRef::from(self.to_ref()).fee_recipient() + } + + pub fn block_hash(&self) -> ExecutionBlockHash { + ExecutionPayloadRef::from(self.to_ref()).block_hash() + } + + pub fn block_number(&self) -> u64 { + ExecutionPayloadRef::from(self.to_ref()).block_number() + } } impl<'a, T: EthSpec> From> for ExecutionPayloadRef<'a, T> { @@ -514,8 +561,8 @@ impl ExecutionPayloadBodyV1 { block_hash: header.block_hash, transactions: self.transactions, withdrawals, - data_gas_used: header.data_gas_used, - excess_data_gas: header.excess_data_gas, + blob_gas_used: header.blob_gas_used, + excess_blob_gas: header.excess_blob_gas, })) } else { Err(format!( @@ -535,6 +582,110 @@ pub struct BlobsBundleV1 { pub blobs: Blobs, } +#[superstruct( + variants(Merge, Capella, Deneb), + variant_attributes(derive(Clone, Debug, PartialEq),), + map_into(ExecutionPayload), + map_ref_into(ExecutionPayloadRef), + cast_error( + ty = "BeaconStateError", + expr = "BeaconStateError::IncorrectStateVariant" + ), + partial_getter_error( + ty = "BeaconStateError", + expr = "BeaconStateError::IncorrectStateVariant" + ) +)] +#[derive(Clone, Debug, PartialEq)] +pub struct NewPayloadRequest { + #[superstruct(only(Merge), partial_getter(rename = "execution_payload_merge"))] + pub execution_payload: ExecutionPayloadMerge, + #[superstruct(only(Capella), partial_getter(rename = "execution_payload_capella"))] + pub execution_payload: ExecutionPayloadCapella, + #[superstruct(only(Deneb), partial_getter(rename = "execution_payload_deneb"))] + pub execution_payload: ExecutionPayloadDeneb, + #[superstruct(only(Deneb))] + pub versioned_hashes: Vec, + #[superstruct(only(Deneb))] + pub parent_beacon_block_root: Hash256, +} + +impl NewPayloadRequest { + pub fn parent_hash(&self) -> ExecutionBlockHash { + match self { + Self::Merge(payload) => payload.execution_payload.parent_hash, + Self::Capella(payload) => payload.execution_payload.parent_hash, + Self::Deneb(payload) => payload.execution_payload.parent_hash, + } + } + + pub fn block_hash(&self) -> ExecutionBlockHash { + match self { + Self::Merge(payload) => payload.execution_payload.block_hash, + Self::Capella(payload) => payload.execution_payload.block_hash, + Self::Deneb(payload) => payload.execution_payload.block_hash, + } + } + + pub fn block_number(&self) -> u64 { + match self { + Self::Merge(payload) => payload.execution_payload.block_number, + Self::Capella(payload) => payload.execution_payload.block_number, + Self::Deneb(payload) => payload.execution_payload.block_number, + } + } + + pub fn into_execution_payload(self) -> ExecutionPayload { + map_new_payload_request_into_execution_payload!(self, |request, cons| { + cons(request.execution_payload) + }) + } +} + +impl<'a, E: EthSpec> TryFrom> for NewPayloadRequest { + type Error = BeaconStateError; + + fn try_from(block: BeaconBlockRef<'a, E>) -> Result { + match block { + BeaconBlockRef::Base(_) | BeaconBlockRef::Altair(_) => { + Err(Self::Error::IncorrectStateVariant) + } + BeaconBlockRef::Merge(block_ref) => Ok(Self::Merge(NewPayloadRequestMerge { + execution_payload: block_ref.body.execution_payload.execution_payload.clone(), + })), + BeaconBlockRef::Capella(block_ref) => Ok(Self::Capella(NewPayloadRequestCapella { + execution_payload: block_ref.body.execution_payload.execution_payload.clone(), + })), + BeaconBlockRef::Deneb(block_ref) => Ok(Self::Deneb(NewPayloadRequestDeneb { + execution_payload: block_ref.body.execution_payload.execution_payload.clone(), + versioned_hashes: block_ref + .body + .blob_kzg_commitments + .iter() + .map(kzg_commitment_to_versioned_hash) + .collect(), + parent_beacon_block_root: block_ref.parent_root, + })), + } + } +} + +impl TryFrom> for NewPayloadRequest { + type Error = BeaconStateError; + + fn try_from(payload: ExecutionPayload) -> Result { + match payload { + ExecutionPayload::Merge(payload) => Ok(Self::Merge(NewPayloadRequestMerge { + execution_payload: payload, + })), + ExecutionPayload::Capella(payload) => Ok(Self::Capella(NewPayloadRequestCapella { + execution_payload: payload, + })), + ExecutionPayload::Deneb(_) => Err(Self::Error::IncorrectStateVariant), + } + } +} + #[derive(Clone, Copy, Debug)] pub struct EngineCapabilities { pub new_payload_v1: bool, @@ -542,6 +693,7 @@ pub struct EngineCapabilities { pub new_payload_v3: bool, pub forkchoice_updated_v1: bool, pub forkchoice_updated_v2: bool, + pub forkchoice_updated_v3: bool, pub get_payload_bodies_by_hash_v1: bool, pub get_payload_bodies_by_range_v1: bool, pub get_payload_v1: bool, @@ -567,6 +719,9 @@ impl EngineCapabilities { if self.forkchoice_updated_v2 { response.push(ENGINE_FORKCHOICE_UPDATED_V2); } + if self.forkchoice_updated_v3 { + response.push(ENGINE_FORKCHOICE_UPDATED_V3); + } if self.get_payload_bodies_by_hash_v1 { response.push(ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1); } diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index 72e0ec92a9d..3eb79d31639 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -11,7 +11,7 @@ use std::collections::HashSet; use tokio::sync::Mutex; use std::time::{Duration, Instant}; -use types::{EthSpec, VersionedHash}; +use types::EthSpec; pub use deposit_log::{DepositLog, Log}; pub use reqwest::Client; @@ -42,6 +42,7 @@ pub const ENGINE_GET_PAYLOAD_TIMEOUT: Duration = Duration::from_secs(2); pub const ENGINE_FORKCHOICE_UPDATED_V1: &str = "engine_forkchoiceUpdatedV1"; pub const ENGINE_FORKCHOICE_UPDATED_V2: &str = "engine_forkchoiceUpdatedV2"; +pub const ENGINE_FORKCHOICE_UPDATED_V3: &str = "engine_forkchoiceUpdatedV3"; pub const ENGINE_FORKCHOICE_UPDATED_TIMEOUT: Duration = Duration::from_secs(8); pub const ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1: &str = "engine_getPayloadBodiesByHashV1"; @@ -80,6 +81,7 @@ pub static PRE_CAPELLA_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilit new_payload_v3: false, forkchoice_updated_v1: true, forkchoice_updated_v2: false, + forkchoice_updated_v3: false, get_payload_bodies_by_hash_v1: false, get_payload_bodies_by_range_v1: false, get_payload_v1: true, @@ -801,12 +803,12 @@ impl HttpJsonRpc { pub async fn new_payload_v3( &self, - execution_payload: ExecutionPayloadDeneb, - versioned_hashes: Vec, + new_payload_request_deneb: NewPayloadRequestDeneb, ) -> Result { let params = json!([ - JsonExecutionPayload::V3(execution_payload.into()), - versioned_hashes + JsonExecutionPayload::V3(new_payload_request_deneb.execution_payload.into()), + new_payload_request_deneb.versioned_hashes, + new_payload_request_deneb.parent_beacon_block_root, ]); let response: JsonPayloadStatusV1 = self @@ -944,6 +946,27 @@ impl HttpJsonRpc { Ok(response.into()) } + pub async fn forkchoice_updated_v3( + &self, + forkchoice_state: ForkchoiceState, + payload_attributes: Option, + ) -> Result { + let params = json!([ + JsonForkchoiceStateV1::from(forkchoice_state), + payload_attributes.map(JsonPayloadAttributes::from) + ]); + + let response: JsonForkchoiceUpdatedV1Response = self + .rpc_request( + ENGINE_FORKCHOICE_UPDATED_V3, + params, + ENGINE_FORKCHOICE_UPDATED_TIMEOUT * self.execution_timeout_multiplier, + ) + .await?; + + Ok(response.into()) + } + pub async fn get_payload_bodies_by_hash_v1( &self, block_hashes: Vec, @@ -1013,6 +1036,7 @@ impl HttpJsonRpc { new_payload_v3: capabilities.contains(ENGINE_NEW_PAYLOAD_V3), forkchoice_updated_v1: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V1), forkchoice_updated_v2: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V2), + forkchoice_updated_v3: capabilities.contains(ENGINE_FORKCHOICE_UPDATED_V3), get_payload_bodies_by_hash_v1: capabilities .contains(ENGINE_GET_PAYLOAD_BODIES_BY_HASH_V1), get_payload_bodies_by_range_v1: capabilities @@ -1056,27 +1080,24 @@ impl HttpJsonRpc { // new_payload that the execution engine supports pub async fn new_payload( &self, - execution_payload: ExecutionPayload, - versioned_hashes_opt: Option>, + new_payload_request: NewPayloadRequest, ) -> Result { let engine_capabilities = self.get_engine_capabilities(None).await?; - match execution_payload { - ExecutionPayload::Merge(_) | ExecutionPayload::Capella(_) => { + match new_payload_request { + NewPayloadRequest::Merge(_) | NewPayloadRequest::Capella(_) => { if engine_capabilities.new_payload_v2 { - self.new_payload_v2(execution_payload).await + self.new_payload_v2(new_payload_request.into_execution_payload()) + .await } else if engine_capabilities.new_payload_v1 { - self.new_payload_v1(execution_payload).await + self.new_payload_v1(new_payload_request.into_execution_payload()) + .await } else { Err(Error::RequiredMethodUnsupported("engine_newPayload")) } } - ExecutionPayload::Deneb(execution_payload_deneb) => { - let Some(versioned_hashes) = versioned_hashes_opt else { - return Err(Error::IncorrectStateVariant); - }; + NewPayloadRequest::Deneb(new_payload_request_deneb) => { if engine_capabilities.new_payload_v3 { - self.new_payload_v3(execution_payload_deneb, versioned_hashes) - .await + self.new_payload_v3(new_payload_request_deneb).await } else { Err(Error::RequiredMethodUnsupported("engine_newPayloadV3")) } @@ -1121,14 +1142,41 @@ impl HttpJsonRpc { pub async fn forkchoice_updated( &self, forkchoice_state: ForkchoiceState, - payload_attributes: Option, + maybe_payload_attributes: Option, ) -> Result { let engine_capabilities = self.get_engine_capabilities(None).await?; - if engine_capabilities.forkchoice_updated_v2 { - self.forkchoice_updated_v2(forkchoice_state, payload_attributes) + if let Some(payload_attributes) = maybe_payload_attributes.as_ref() { + match payload_attributes { + PayloadAttributes::V1(_) | PayloadAttributes::V2(_) => { + if engine_capabilities.forkchoice_updated_v2 { + self.forkchoice_updated_v2(forkchoice_state, maybe_payload_attributes) + .await + } else if engine_capabilities.forkchoice_updated_v1 { + self.forkchoice_updated_v1(forkchoice_state, maybe_payload_attributes) + .await + } else { + Err(Error::RequiredMethodUnsupported("engine_forkchoiceUpdated")) + } + } + PayloadAttributes::V3(_) => { + if engine_capabilities.forkchoice_updated_v3 { + self.forkchoice_updated_v3(forkchoice_state, maybe_payload_attributes) + .await + } else { + Err(Error::RequiredMethodUnsupported( + "engine_forkchoiceUpdatedV3", + )) + } + } + } + } else if engine_capabilities.forkchoice_updated_v3 { + self.forkchoice_updated_v3(forkchoice_state, maybe_payload_attributes) + .await + } else if engine_capabilities.forkchoice_updated_v2 { + self.forkchoice_updated_v2(forkchoice_state, maybe_payload_attributes) .await } else if engine_capabilities.forkchoice_updated_v1 { - self.forkchoice_updated_v1(forkchoice_state, payload_attributes) + self.forkchoice_updated_v1(forkchoice_state, maybe_payload_attributes) .await } else { Err(Error::RequiredMethodUnsupported("engine_forkchoiceUpdated")) diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index d94e7078764..bce4e686f92 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -100,10 +100,10 @@ pub struct JsonExecutionPayload { pub withdrawals: VariableList, #[superstruct(only(V3))] #[serde(with = "serde_utils::u64_hex_be")] - pub data_gas_used: u64, + pub blob_gas_used: u64, #[superstruct(only(V3))] #[serde(with = "serde_utils::u64_hex_be")] - pub excess_data_gas: u64, + pub excess_blob_gas: u64, } impl From> for JsonExecutionPayloadV1 { @@ -175,8 +175,8 @@ impl From> for JsonExecutionPayloadV3 { .map(Into::into) .collect::>() .into(), - data_gas_used: payload.data_gas_used, - excess_data_gas: payload.excess_data_gas, + blob_gas_used: payload.blob_gas_used, + excess_blob_gas: payload.excess_blob_gas, } } } @@ -260,8 +260,8 @@ impl From> for ExecutionPayloadDeneb { .map(Into::into) .collect::>() .into(), - data_gas_used: payload.data_gas_used, - excess_data_gas: payload.excess_data_gas, + blob_gas_used: payload.blob_gas_used, + excess_blob_gas: payload.excess_blob_gas, } } } @@ -298,6 +298,8 @@ pub struct JsonGetPayloadResponse { pub block_value: Uint256, #[superstruct(only(V3))] pub blobs_bundle: JsonBlobsBundleV1, + #[superstruct(only(V3))] + pub should_override_builder: bool, } impl From> for GetPayloadResponse { @@ -320,6 +322,7 @@ impl From> for GetPayloadResponse { execution_payload: response.execution_payload.into(), block_value: response.block_value, blobs_bundle: response.blobs_bundle.into(), + should_override_builder: response.should_override_builder, }) } } @@ -361,7 +364,7 @@ impl From for Withdrawal { } #[superstruct( - variants(V1, V2), + variants(V1, V2, V3), variant_attributes( derive(Debug, Clone, PartialEq, Serialize, Deserialize), serde(rename_all = "camelCase") @@ -376,8 +379,10 @@ pub struct JsonPayloadAttributes { pub timestamp: u64, pub prev_randao: Hash256, pub suggested_fee_recipient: Address, - #[superstruct(only(V2))] + #[superstruct(only(V2, V3))] pub withdrawals: Vec, + #[superstruct(only(V3))] + pub parent_beacon_block_root: Hash256, } impl From for JsonPayloadAttributes { @@ -394,6 +399,13 @@ impl From for JsonPayloadAttributes { suggested_fee_recipient: pa.suggested_fee_recipient, withdrawals: pa.withdrawals.into_iter().map(Into::into).collect(), }), + PayloadAttributes::V3(pa) => Self::V3(JsonPayloadAttributesV3 { + timestamp: pa.timestamp, + prev_randao: pa.prev_randao, + suggested_fee_recipient: pa.suggested_fee_recipient, + withdrawals: pa.withdrawals.into_iter().map(Into::into).collect(), + parent_beacon_block_root: pa.parent_beacon_block_root, + }), } } } @@ -412,6 +424,13 @@ impl From for PayloadAttributes { suggested_fee_recipient: jpa.suggested_fee_recipient, withdrawals: jpa.withdrawals.into_iter().map(Into::into).collect(), }), + JsonPayloadAttributes::V3(jpa) => Self::V3(PayloadAttributesV3 { + timestamp: jpa.timestamp, + prev_randao: jpa.prev_randao, + suggested_fee_recipient: jpa.suggested_fee_recipient, + withdrawals: jpa.withdrawals.into_iter().map(Into::into).collect(), + parent_beacon_block_root: jpa.parent_beacon_block_root, + }), } } } diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index c513606dcbf..e997fc596ae 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -44,10 +44,8 @@ use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::Blobs; use types::KzgProofs; use types::{ - AbstractExecPayload, BeaconStateError, ExecPayload, ExecutionPayloadDeneb, VersionedHash, -}; -use types::{ - BlindedPayload, BlockType, ChainSpec, Epoch, ExecutionPayloadCapella, ExecutionPayloadMerge, + AbstractExecPayload, BeaconStateError, BlindedPayload, BlockType, ChainSpec, Epoch, + ExecPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, }; use types::{ProposerPreparationData, PublicKeyBytes, Signature, Slot, Transaction}; @@ -300,6 +298,7 @@ struct Inner { builder_profit_threshold: Uint256, log: Logger, always_prefer_builder_payload: bool, + ignore_builder_override_suggestion_threshold: f32, /// Track whether the last `newPayload` call errored. /// /// This is used *only* in the informational sync status endpoint, so that a VC using this @@ -330,6 +329,7 @@ pub struct Config { pub builder_profit_threshold: u128, pub execution_timeout_multiplier: Option, pub always_prefer_builder_payload: bool, + pub ignore_builder_override_suggestion_threshold: f32, } /// Provides access to one execution engine and provides a neat interface for consumption by the @@ -339,6 +339,40 @@ pub struct ExecutionLayer { inner: Arc>, } +/// This function will return the percentage difference between 2 U256 values, using `base_value` +/// as the denominator. It is accurate to 7 decimal places which is about the precision of +/// an f32. +/// +/// If some error is encountered in the calculation, None will be returned. +fn percentage_difference_u256(base_value: Uint256, comparison_value: Uint256) -> Option { + if base_value == Uint256::zero() { + return None; + } + // this is the total supply of ETH in WEI + let max_value = Uint256::from(12u8) * Uint256::exp10(25); + if base_value > max_value || comparison_value > max_value { + return None; + } + + // Now we should be able to calculate the difference without division by zero or overflow + const PRECISION: usize = 7; + let precision_factor = Uint256::exp10(PRECISION); + let scaled_difference = if base_value <= comparison_value { + (comparison_value - base_value) * precision_factor + } else { + (base_value - comparison_value) * precision_factor + }; + let scaled_proportion = scaled_difference / base_value; + // max value of scaled difference is 1.2 * 10^33, well below the max value of a u128 / f64 / f32 + let percentage = + 100.0f64 * scaled_proportion.low_u128() as f64 / precision_factor.low_u128() as f64; + if base_value <= comparison_value { + Some(percentage as f32) + } else { + Some(-percentage as f32) + } +} + impl ExecutionLayer { /// Instantiate `Self` with an Execution engine specified in `Config`, using JSON-RPC via HTTP. pub fn from_config(config: Config, executor: TaskExecutor, log: Logger) -> Result { @@ -354,6 +388,7 @@ impl ExecutionLayer { builder_profit_threshold, execution_timeout_multiplier, always_prefer_builder_payload, + ignore_builder_override_suggestion_threshold, } = config; if urls.len() > 1 { @@ -433,6 +468,7 @@ impl ExecutionLayer { builder_profit_threshold: Uint256::from(builder_profit_threshold), log, always_prefer_builder_payload, + ignore_builder_override_suggestion_threshold, last_new_payload_errored: RwLock::new(false), }; @@ -755,7 +791,7 @@ impl ExecutionLayer { current_fork, ) .await - .map(ProvenancedPayload::Local) + .map(|get_payload_response| ProvenancedPayload::Local(get_payload_response.into())) } }; @@ -824,7 +860,7 @@ impl ExecutionLayer { .await }), timed_future(metrics::GET_BLINDED_PAYLOAD_LOCAL, async { - self.get_full_payload_caching::( + self.get_full_payload_caching( parent_hash, payload_attributes, forkchoice_update_params, @@ -844,7 +880,7 @@ impl ExecutionLayer { }, "relay_response_ms" => relay_duration.as_millis(), "local_fee_recipient" => match &local_result { - Ok(proposal_contents) => format!("{:?}", proposal_contents.payload().fee_recipient()), + Ok(get_payload_response) => format!("{:?}", get_payload_response.fee_recipient()), Err(_) => "request failed".to_string() }, "local_response_ms" => local_duration.as_millis(), @@ -858,20 +894,20 @@ impl ExecutionLayer { "Builder error when requesting payload"; "info" => "falling back to local execution client", "relay_error" => ?e, - "local_block_hash" => ?local.payload().block_hash(), + "local_block_hash" => ?local.block_hash(), "parent_hash" => ?parent_hash, ); - Ok(ProvenancedPayload::Local(local)) + Ok(ProvenancedPayload::Local(local.into())) } (Ok(None), Ok(local)) => { info!( self.log(), "Builder did not return a payload"; "info" => "falling back to local execution client", - "local_block_hash" => ?local.payload().block_hash(), + "local_block_hash" => ?local.block_hash(), "parent_hash" => ?parent_hash, ); - Ok(ProvenancedPayload::Local(local)) + Ok(ProvenancedPayload::Local(local.into())) } (Ok(Some(relay)), Ok(local)) => { let header = &relay.data.message.header; @@ -880,12 +916,13 @@ impl ExecutionLayer { self.log(), "Received local and builder payloads"; "relay_block_hash" => ?header.block_hash(), - "local_block_hash" => ?local.payload().block_hash(), + "local_block_hash" => ?local.block_hash(), "parent_hash" => ?parent_hash, ); let relay_value = relay.data.message.value; let local_value = *local.block_value(); + if !self.inner.always_prefer_builder_payload { if local_value >= relay_value { info!( @@ -894,7 +931,24 @@ impl ExecutionLayer { "local_block_value" => %local_value, "relay_value" => %relay_value ); - return Ok(ProvenancedPayload::Local(local)); + return Ok(ProvenancedPayload::Local(local.into())); + } else if local.should_override_builder().unwrap_or(false) { + let percentage_difference = + percentage_difference_u256(local_value, relay_value); + if percentage_difference.map_or(false, |percentage| { + percentage + < self + .inner + .ignore_builder_override_suggestion_threshold + }) { + info!( + self.log(), + "Using local payload because execution engine suggested we ignore builder payload"; + "local_block_value" => %local_value, + "relay_value" => %relay_value + ); + return Ok(ProvenancedPayload::Local(local.into())); + } } else { info!( self.log(), @@ -909,7 +963,7 @@ impl ExecutionLayer { &relay, parent_hash, payload_attributes, - Some(local.payload().block_number()), + Some(local.block_number()), self.inner.builder_profit_threshold, current_fork, spec, @@ -929,7 +983,7 @@ impl ExecutionLayer { "relay_block_hash" => ?header.block_hash(), "parent_hash" => ?parent_hash, ); - Ok(ProvenancedPayload::Local(local)) + Ok(ProvenancedPayload::Local(local.into())) } Err(reason) => { metrics::inc_counter_vec( @@ -944,7 +998,7 @@ impl ExecutionLayer { "relay_block_hash" => ?header.block_hash(), "parent_hash" => ?parent_hash, ); - Ok(ProvenancedPayload::Local(local)) + Ok(ProvenancedPayload::Local(local.into())) } } } @@ -1049,17 +1103,17 @@ impl ExecutionLayer { current_fork, ) .await - .map(ProvenancedPayload::Local) + .map(|get_payload_response| ProvenancedPayload::Local(get_payload_response.into())) } /// Get a full payload without caching its result in the execution layer's payload cache. - async fn get_full_payload>( + async fn get_full_payload( &self, parent_hash: ExecutionBlockHash, payload_attributes: &PayloadAttributes, forkchoice_update_params: ForkchoiceUpdateParameters, current_fork: ForkName, - ) -> Result, Error> { + ) -> Result, Error> { self.get_full_payload_with( parent_hash, payload_attributes, @@ -1071,13 +1125,13 @@ impl ExecutionLayer { } /// Get a full payload and cache its result in the execution layer's payload cache. - async fn get_full_payload_caching>( + async fn get_full_payload_caching( &self, parent_hash: ExecutionBlockHash, payload_attributes: &PayloadAttributes, forkchoice_update_params: ForkchoiceUpdateParameters, current_fork: ForkName, - ) -> Result, Error> { + ) -> Result, Error> { self.get_full_payload_with( parent_hash, payload_attributes, @@ -1088,14 +1142,14 @@ impl ExecutionLayer { .await } - async fn get_full_payload_with>( + async fn get_full_payload_with( &self, parent_hash: ExecutionBlockHash, payload_attributes: &PayloadAttributes, forkchoice_update_params: ForkchoiceUpdateParameters, current_fork: ForkName, f: fn(&ExecutionLayer, ExecutionPayloadRef) -> Option>, - ) -> Result, Error> { + ) -> Result, Error> { self.engine() .request(move |engine| async move { let payload_id = if let Some(id) = engine @@ -1181,7 +1235,7 @@ impl ExecutionLayer { ); } - Ok(payload_response.into()) + Ok(payload_response) }) .await .map_err(Box::new) @@ -1191,29 +1245,25 @@ impl ExecutionLayer { /// Maps to the `engine_newPayload` JSON-RPC call. pub async fn notify_new_payload( &self, - execution_payload: &ExecutionPayload, - versioned_hashes: Option>, + new_payload_request: NewPayloadRequest, ) -> Result { let _timer = metrics::start_timer_vec( &metrics::EXECUTION_LAYER_REQUEST_TIMES, &[metrics::NEW_PAYLOAD], ); + let block_hash = new_payload_request.block_hash(); trace!( self.log(), "Issuing engine_newPayload"; - "parent_hash" => ?execution_payload.parent_hash(), - "block_hash" => ?execution_payload.block_hash(), - "block_number" => execution_payload.block_number(), + "parent_hash" => ?new_payload_request.parent_hash(), + "block_hash" => ?block_hash, + "block_number" => ?new_payload_request.block_number(), ); let result = self .engine() - .request(|engine| { - engine - .api - .new_payload(execution_payload.clone(), versioned_hashes) - }) + .request(|engine| engine.api.new_payload(new_payload_request)) .await; if let Ok(status) = &result { @@ -1224,7 +1274,7 @@ impl ExecutionLayer { } *self.inner.last_new_payload_errored.write().await = result.is_err(); - process_payload_status(execution_payload.block_hash(), result, self.log()) + process_payload_status(block_hash, result, self.log()) .map_err(Box::new) .map_err(Error::EngineError) } @@ -1796,8 +1846,8 @@ impl ExecutionLayer { block_hash: deneb_block.block_hash, transactions: convert_transactions(deneb_block.transactions)?, withdrawals, - data_gas_used: deneb_block.data_gas_used, - excess_data_gas: deneb_block.excess_data_gas, + blob_gas_used: deneb_block.blob_gas_used, + excess_blob_gas: deneb_block.excess_blob_gas, }) } }; @@ -2297,4 +2347,42 @@ mod test { }) .await; } + + #[tokio::test] + async fn percentage_difference_u256_tests() { + // ensure function returns `None` when base value is zero + assert_eq!(percentage_difference_u256(0.into(), 1.into()), None); + // ensure function returns `None` when either value is greater than 120 Million ETH + let max_value = Uint256::from(12u8) * Uint256::exp10(25); + assert_eq!( + percentage_difference_u256(1u8.into(), max_value + Uint256::from(1u8)), + None + ); + assert_eq!( + percentage_difference_u256(max_value + Uint256::from(1u8), 1u8.into()), + None + ); + // it should work up to max value + assert_eq!( + percentage_difference_u256(max_value, max_value / Uint256::from(2u8)), + Some(-50f32) + ); + // should work when base value is greater than comparison value + assert_eq!( + percentage_difference_u256(4u8.into(), 3u8.into()), + Some(-25f32) + ); + // should work when comparison value is greater than base value + assert_eq!( + percentage_difference_u256(4u8.into(), 5u8.into()), + Some(25f32) + ); + // should be accurate to 7 decimal places + let result = + percentage_difference_u256(Uint256::from(31415926u64), Uint256::from(13371337u64)) + .expect("should get percentage"); + // result = -57.4377116 + assert!(result > -57.43772); + assert!(result <= -57.43771); + } } diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 937c6d9da33..e6fa17349c7 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -556,27 +556,27 @@ impl ExecutionBlockGenerator { transactions: vec![].into(), withdrawals: pa.withdrawals.clone().into(), }), - ForkName::Deneb => ExecutionPayload::Deneb(ExecutionPayloadDeneb { - parent_hash: forkchoice_state.head_block_hash, - fee_recipient: pa.suggested_fee_recipient, - receipts_root: Hash256::repeat_byte(42), - state_root: Hash256::repeat_byte(43), - logs_bloom: vec![0; 256].into(), - prev_randao: pa.prev_randao, - block_number: parent.block_number() + 1, - gas_limit: GAS_LIMIT, - gas_used: GAS_USED, - timestamp: pa.timestamp, - extra_data: "block gen was here".as_bytes().to_vec().into(), - base_fee_per_gas: Uint256::one(), - block_hash: ExecutionBlockHash::zero(), - transactions: vec![].into(), - withdrawals: pa.withdrawals.clone().into(), - data_gas_used: 0, - excess_data_gas: 0, - }), _ => unreachable!(), }, + PayloadAttributes::V3(pa) => ExecutionPayload::Deneb(ExecutionPayloadDeneb { + parent_hash: forkchoice_state.head_block_hash, + fee_recipient: pa.suggested_fee_recipient, + receipts_root: Hash256::repeat_byte(42), + state_root: Hash256::repeat_byte(43), + logs_bloom: vec![0; 256].into(), + prev_randao: pa.prev_randao, + block_number: parent.block_number() + 1, + gas_limit: GAS_LIMIT, + gas_used: GAS_USED, + timestamp: pa.timestamp, + extra_data: "block gen was here".as_bytes().to_vec().into(), + base_fee_per_gas: Uint256::one(), + block_hash: ExecutionBlockHash::zero(), + transactions: vec![].into(), + withdrawals: pa.withdrawals.clone().into(), + blob_gas_used: 0, + excess_blob_gas: 0, + }), }; match execution_payload.fork_name() { diff --git a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs index 7ed954a30a6..e50e6f8d379 100644 --- a/beacon_node/execution_layer/src/test_utils/handle_rpc.rs +++ b/beacon_node/execution_layer/src/test_utils/handle_rpc.rs @@ -306,6 +306,7 @@ pub async fn handle_rpc( GENERIC_ERROR_CODE, ))? .into(), + should_override_builder: false, }) .unwrap() } @@ -313,7 +314,9 @@ pub async fn handle_rpc( _ => unreachable!(), } } - ENGINE_FORKCHOICE_UPDATED_V1 | ENGINE_FORKCHOICE_UPDATED_V2 => { + ENGINE_FORKCHOICE_UPDATED_V1 + | ENGINE_FORKCHOICE_UPDATED_V2 + | ENGINE_FORKCHOICE_UPDATED_V3 => { let forkchoice_state: JsonForkchoiceStateV1 = get_param(params, 0).map_err(|s| (s, BAD_PARAMS_ERROR_CODE))?; let payload_attributes = match method { @@ -351,10 +354,15 @@ pub async fn handle_rpc( }) .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))? } + ENGINE_FORKCHOICE_UPDATED_V3 => { + get_param::>(params, 1) + .map(|opt| opt.map(JsonPayloadAttributes::V3)) + .map_err(|s| (s, BAD_PARAMS_ERROR_CODE))? + } _ => unreachable!(), }; - // validate method called correctly according to shanghai fork time + // validate method called correctly according to fork time if let Some(pa) = payload_attributes.as_ref() { match ctx .execution_block_generator @@ -372,13 +380,22 @@ pub async fn handle_rpc( )); } } - ForkName::Capella | ForkName::Deneb => { + ForkName::Capella => { if method == ENGINE_FORKCHOICE_UPDATED_V1 { return Err(( format!("{} called after Capella fork!", method), FORK_REQUEST_MISMATCH_ERROR_CODE, )); } + if method == ENGINE_FORKCHOICE_UPDATED_V3 { + return Err(( + format!( + "{} called with `JsonPayloadAttributesV3` before Deneb fork!", + method + ), + GENERIC_ERROR_CODE, + )); + } if matches!(pa, JsonPayloadAttributes::V1(_)) { return Err(( format!( @@ -389,6 +406,20 @@ pub async fn handle_rpc( )); } } + ForkName::Deneb => { + if method == ENGINE_FORKCHOICE_UPDATED_V1 { + return Err(( + format!("{} called after Deneb fork!", method), + FORK_REQUEST_MISMATCH_ERROR_CODE, + )); + } + if method == ENGINE_FORKCHOICE_UPDATED_V2 { + return Err(( + format!("{} called after Deneb fork!", method), + FORK_REQUEST_MISMATCH_ERROR_CODE, + )); + } + } _ => unreachable!(), }; } diff --git a/beacon_node/execution_layer/src/test_utils/mock_builder.rs b/beacon_node/execution_layer/src/test_utils/mock_builder.rs index 7a28e694297..cd29a998b0f 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_builder.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_builder.rs @@ -36,8 +36,8 @@ use task_executor::TaskExecutor; use tempfile::NamedTempFile; use tree_hash::TreeHash; use types::{ - Address, BeaconState, BlindedPayload, ChainSpec, EthSpec, ExecPayload, ForkName, Hash256, Slot, - Uint256, + Address, BeaconState, ChainSpec, EthSpec, ExecPayload, ExecutionPayload, + ExecutionPayloadHeader, ForkName, Hash256, Slot, Uint256, }; #[derive(Clone)] @@ -402,13 +402,23 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { let prev_randao = head_state .get_randao_mix(head_state.current_epoch()) .map_err(convert_err)?; + let parent_root = head_state.latest_block_header().parent_root; let payload_attributes = match fork { - ForkName::Merge => PayloadAttributes::new(timestamp, *prev_randao, fee_recipient, None), + ForkName::Merge => { + PayloadAttributes::new(timestamp, *prev_randao, fee_recipient, None, None) + } // the withdrawals root is filled in by operations - ForkName::Capella | ForkName::Deneb => { - PayloadAttributes::new(timestamp, *prev_randao, fee_recipient, Some(vec![])) + ForkName::Capella => { + PayloadAttributes::new(timestamp, *prev_randao, fee_recipient, Some(vec![]), None) } + ForkName::Deneb => PayloadAttributes::new( + timestamp, + *prev_randao, + fee_recipient, + Some(vec![]), + Some(parent_root), + ), ForkName::Base | ForkName::Altair => { return Err(MevError::InvalidFork); } @@ -425,9 +435,9 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { finalized_hash: Some(finalized_execution_hash), }; - let payload = self + let payload: ExecutionPayload = self .el - .get_full_payload_caching::>( + .get_full_payload_caching( head_execution_hash, &payload_attributes, forkchoice_update_params, @@ -435,10 +445,17 @@ impl mev_rs::BlindedBlockProvider for MockBuilder { ) .await .map_err(convert_err)? - .to_payload() - .to_execution_payload_header(); + .into(); - let json_payload = serde_json::to_string(&payload).map_err(convert_err)?; + let header: ExecutionPayloadHeader = match payload { + ExecutionPayload::Merge(payload) => ExecutionPayloadHeader::Merge((&payload).into()), + ExecutionPayload::Capella(payload) => { + ExecutionPayloadHeader::Capella((&payload).into()) + } + ExecutionPayload::Deneb(payload) => ExecutionPayloadHeader::Deneb((&payload).into()), + }; + + let json_payload = serde_json::to_string(&header).map_err(convert_err)?; let mut message = match fork { ForkName::Capella => BuilderBid::Capella(BuilderBidCapella { header: serde_json::from_str(json_payload.as_str()).map_err(convert_err)?, diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index b4a7d247adf..d82aca3bc40 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -110,7 +110,8 @@ impl MockExecutionLayer { timestamp, prev_randao, Address::repeat_byte(42), - // FIXME: think about how to handle different forks / withdrawals here.. + // FIXME: think about how to handle different forks here.. + None, None, ); @@ -140,7 +141,7 @@ impl MockExecutionLayer { }; let suggested_fee_recipient = self.el.get_suggested_fee_recipient(validator_index).await; let payload_attributes = - PayloadAttributes::new(timestamp, prev_randao, suggested_fee_recipient, None); + PayloadAttributes::new(timestamp, prev_randao, suggested_fee_recipient, None, None); let payload: ExecutionPayload = self .el .get_payload::>( @@ -148,7 +149,7 @@ impl MockExecutionLayer { &payload_attributes, forkchoice_update_params, builder_params, - // FIXME: do we need to consider other forks somehow? What about withdrawals? + // FIXME: do we need to consider other forks somehow? ForkName::Merge, &self.spec, ) @@ -175,7 +176,7 @@ impl MockExecutionLayer { }; let suggested_fee_recipient = self.el.get_suggested_fee_recipient(validator_index).await; let payload_attributes = - PayloadAttributes::new(timestamp, prev_randao, suggested_fee_recipient, None); + PayloadAttributes::new(timestamp, prev_randao, suggested_fee_recipient, None, None); let payload_header = self .el .get_payload::>( @@ -204,7 +205,12 @@ impl MockExecutionLayer { Some(payload.clone()) ); - let status = self.el.notify_new_payload(&payload, None).await.unwrap(); + // TODO: again consider forks + let status = self + .el + .notify_new_payload(payload.try_into().unwrap()) + .await + .unwrap(); assert_eq!(status, PayloadStatus::Valid); // Use junk values for slot/head-root to ensure there is no payload supplied. diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index eb37d0c5ee1..95abbdbc90f 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -44,6 +44,7 @@ pub const DEFAULT_ENGINE_CAPABILITIES: EngineCapabilities = EngineCapabilities { new_payload_v3: true, forkchoice_updated_v1: true, forkchoice_updated_v2: true, + forkchoice_updated_v3: true, get_payload_bodies_by_hash_v1: true, get_payload_bodies_by_range_v1: true, get_payload_v1: true, diff --git a/beacon_node/operation_pool/src/attestation.rs b/beacon_node/operation_pool/src/attestation.rs index fbbd5d7ddcf..97c291aa855 100644 --- a/beacon_node/operation_pool/src/attestation.rs +++ b/beacon_node/operation_pool/src/attestation.rs @@ -30,7 +30,7 @@ impl<'a, T: EthSpec> AttMaxCover<'a, T> { if let BeaconState::Base(ref base_state) = state { Self::new_for_base(att, state, base_state, total_active_balance, spec) } else { - Self::new_for_altair(att, state, reward_cache, total_active_balance, spec) + Self::new_for_altair_deneb(att, state, reward_cache, total_active_balance, spec) } } @@ -69,7 +69,7 @@ impl<'a, T: EthSpec> AttMaxCover<'a, T> { } /// Initialise an attestation cover object for Altair or later. - pub fn new_for_altair( + pub fn new_for_altair_deneb( att: AttestationRef<'a, T>, state: &BeaconState, reward_cache: &'a RewardCache, diff --git a/beacon_node/operation_pool/src/lib.rs b/beacon_node/operation_pool/src/lib.rs index 24c0623f5c3..7e1ddb1fd2f 100644 --- a/beacon_node/operation_pool/src/lib.rs +++ b/beacon_node/operation_pool/src/lib.rs @@ -1852,7 +1852,21 @@ mod release_tests { // Sign an exit with the Altair domain and a phase0 epoch. This is a weird type of exit // that is valid because after the Bellatrix fork we'll use the Altair fork domain to verify // all prior epochs. - let exit2 = harness.make_voluntary_exit(2, Epoch::new(0)); + let unsigned_exit = VoluntaryExit { + epoch: Epoch::new(0), + validator_index: 2, + }; + let exit2 = SignedVoluntaryExit { + message: unsigned_exit.clone(), + signature: harness.validator_keypairs[2] + .sk + .sign(unsigned_exit.signing_root(spec.compute_domain( + Domain::VoluntaryExit, + harness.spec.altair_fork_version, + harness.chain.genesis_validators_root, + ))), + }; + let verified_exit2 = exit2 .clone() .validate(&bellatrix_head.beacon_state, &harness.chain.spec) diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 9b4b4630743..44580e4a518 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -1089,6 +1089,23 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .default_value("0") .takes_value(true) ) + .arg( + Arg::with_name("ignore-builder-override-suggestion-threshold") + .long("ignore-builder-override-suggestion-threshold") + .value_name("PERCENTAGE") + .help("When the EE advises Lighthouse to ignore the builder payload, this flag \ + specifies a percentage threshold for the difference between the reward from \ + the builder payload and the local EE's payload. This threshold must be met \ + for Lighthouse to consider ignoring the EE's suggestion. If the reward from \ + the builder's payload doesn't exceed the local payload by at least this \ + percentage, the local payload will be used. The conditions under which the \ + EE may make this suggestion depend on the EE's implementation, with the \ + primary intent being to safeguard against potential censorship attacks \ + from builders. Setting this flag to 0 will cause Lighthouse to always \ + ignore the EE's suggestion. Default: 10.0 (equivalent to 10%).") + .default_value("10.0") + .takes_value(true) + ) .arg( Arg::with_name("builder-user-agent") .long("builder-user-agent") @@ -1160,6 +1177,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { // to local payloads, therefore it fundamentally conflicts with // always using the builder. .conflicts_with("builder-profit-threshold") + .conflicts_with("ignore-builder-override-suggestion-threshold") ) .arg( Arg::with_name("invalid-gossip-verified-blocks-path") diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index fe0b163603e..3c5f6298414 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -345,6 +345,8 @@ pub fn get_config( el_config.default_datadir = client_config.data_dir().clone(); el_config.builder_profit_threshold = clap_utils::parse_required(cli_args, "builder-profit-threshold")?; + el_config.ignore_builder_override_suggestion_threshold = + clap_utils::parse_required(cli_args, "ignore-builder-override-suggestion-threshold")?; let execution_timeout_multiplier = clap_utils::parse_required(cli_args, "execution-timeout-multiplier")?; el_config.execution_timeout_multiplier = Some(execution_timeout_multiplier); diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 60ee3041037..cba01fa2660 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -926,7 +926,7 @@ pub struct SseLateHead { } #[superstruct( - variants(V1, V2), + variants(V1, V2, V3), variant_attributes(derive(Clone, Debug, Eq, Hash, PartialEq, Serialize, Deserialize)) )] #[derive(Clone, Debug, Eq, Hash, PartialEq, Deserialize, Serialize)] @@ -939,8 +939,10 @@ pub struct SsePayloadAttributes { pub prev_randao: Hash256, #[superstruct(getter(copy))] pub suggested_fee_recipient: Address, - #[superstruct(only(V2))] + #[superstruct(only(V2, V3))] pub withdrawals: Vec, + #[superstruct(only(V3), partial_getter(copy))] + pub parent_beacon_block_root: Hash256, } #[derive(PartialEq, Debug, Deserialize, Serialize, Clone)] diff --git a/consensus/state_processing/src/common/get_attestation_participation.rs b/consensus/state_processing/src/common/get_attestation_participation.rs index 499d8fa8f86..e4e30230af5 100644 --- a/consensus/state_processing/src/common/get_attestation_participation.rs +++ b/consensus/state_processing/src/common/get_attestation_participation.rs @@ -44,8 +44,21 @@ pub fn get_attestation_participation_flag_indices( if is_matching_source && inclusion_delay <= T::slots_per_epoch().integer_sqrt() { participation_flag_indices.push(TIMELY_SOURCE_FLAG_INDEX); } - if is_matching_target && inclusion_delay <= T::slots_per_epoch() { - participation_flag_indices.push(TIMELY_TARGET_FLAG_INDEX); + match state { + &BeaconState::Base(_) + | &BeaconState::Altair(_) + | &BeaconState::Merge(_) + | &BeaconState::Capella(_) => { + if is_matching_target && inclusion_delay <= T::slots_per_epoch() { + participation_flag_indices.push(TIMELY_TARGET_FLAG_INDEX); + } + } + &BeaconState::Deneb(_) => { + if is_matching_target { + // [Modified in Deneb:EIP7045] + participation_flag_indices.push(TIMELY_TARGET_FLAG_INDEX); + } + } } if is_matching_head && inclusion_delay == spec.min_attestation_inclusion_delay { participation_flag_indices.push(TIMELY_HEAD_FLAG_INDEX); diff --git a/consensus/state_processing/src/per_block_processing/process_operations.rs b/consensus/state_processing/src/per_block_processing/process_operations.rs index 4e60c416184..cb24a7ba7ec 100644 --- a/consensus/state_processing/src/per_block_processing/process_operations.rs +++ b/consensus/state_processing/src/per_block_processing/process_operations.rs @@ -95,7 +95,7 @@ pub mod base { } } -pub mod altair { +pub mod altair_deneb { use super::*; use crate::common::update_progressive_balances_cache::update_progressive_balances_on_attestation; use types::consts::altair::TIMELY_TARGET_FLAG_INDEX; @@ -269,7 +269,7 @@ pub fn process_attestations>( | BeaconBlockBodyRef::Merge(_) | BeaconBlockBodyRef::Capella(_) | BeaconBlockBodyRef::Deneb(_) => { - altair::process_attestations( + altair_deneb::process_attestations( state, block_body.attestations(), verify_signatures, diff --git a/consensus/state_processing/src/per_block_processing/signature_sets.rs b/consensus/state_processing/src/per_block_processing/signature_sets.rs index c05d3f057d7..fcd324e9eb1 100644 --- a/consensus/state_processing/src/per_block_processing/signature_sets.rs +++ b/consensus/state_processing/src/per_block_processing/signature_sets.rs @@ -387,12 +387,23 @@ where let exit = &signed_exit.message; let proposer_index = exit.validator_index as usize; - let domain = spec.get_domain( - exit.epoch, - Domain::VoluntaryExit, - &state.fork(), - state.genesis_validators_root(), - ); + let domain = match state { + BeaconState::Base(_) + | BeaconState::Altair(_) + | BeaconState::Merge(_) + | BeaconState::Capella(_) => spec.get_domain( + exit.epoch, + Domain::VoluntaryExit, + &state.fork(), + state.genesis_validators_root(), + ), + // EIP-7044 + BeaconState::Deneb(_) => spec.compute_domain( + Domain::VoluntaryExit, + spec.capella_fork_version, + state.genesis_validators_root(), + ), + }; let message = exit.signing_root(domain); diff --git a/consensus/state_processing/src/per_block_processing/verify_attestation.rs b/consensus/state_processing/src/per_block_processing/verify_attestation.rs index 303a6e3913a..b7aa4643e48 100644 --- a/consensus/state_processing/src/per_block_processing/verify_attestation.rs +++ b/consensus/state_processing/src/per_block_processing/verify_attestation.rs @@ -32,13 +32,22 @@ pub fn verify_attestation_for_block_inclusion<'ctxt, T: EthSpec>( attestation: data.slot, } ); - verify!( - state.slot() <= data.slot.safe_add(T::slots_per_epoch())?, - Invalid::IncludedTooLate { - state: state.slot(), - attestation: data.slot, + match state { + BeaconState::Base(_) + | BeaconState::Altair(_) + | BeaconState::Merge(_) + | BeaconState::Capella(_) => { + verify!( + state.slot() <= data.slot.safe_add(T::slots_per_epoch())?, + Invalid::IncludedTooLate { + state: state.slot(), + attestation: data.slot, + } + ); } - ); + // [Modified in Deneb:EIP7045] + BeaconState::Deneb(_) => {} + } verify_attestation_for_state(state, attestation, ctxt, verify_signatures, spec) } diff --git a/consensus/types/src/execution_block_header.rs b/consensus/types/src/execution_block_header.rs index 9dca6797399..5ec5484cab4 100644 --- a/consensus/types/src/execution_block_header.rs +++ b/consensus/types/src/execution_block_header.rs @@ -26,8 +26,9 @@ use metastruct::metastruct; #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[metastruct(mappings(map_execution_block_header_fields_except_withdrawals(exclude( withdrawals_root, - data_gas_used, - excess_data_gas + blob_gas_used, + excess_blob_gas, + parent_beacon_block_root )),))] pub struct ExecutionBlockHeader { pub parent_hash: Hash256, @@ -47,8 +48,9 @@ pub struct ExecutionBlockHeader { pub nonce: Hash64, pub base_fee_per_gas: Uint256, pub withdrawals_root: Option, - pub data_gas_used: Option, - pub excess_data_gas: Option, + pub blob_gas_used: Option, + pub excess_blob_gas: Option, + pub parent_beacon_block_root: Option, } impl ExecutionBlockHeader { @@ -57,8 +59,9 @@ impl ExecutionBlockHeader { rlp_empty_list_root: Hash256, rlp_transactions_root: Hash256, rlp_withdrawals_root: Option, - rlp_data_gas_used: Option, - rlp_excess_data_gas: Option, + rlp_blob_gas_used: Option, + rlp_excess_blob_gas: Option, + rlp_parent_beacon_block_root: Option, ) -> Self { // Most of these field mappings are defined in EIP-3675 except for `mixHash`, which is // defined in EIP-4399. @@ -80,8 +83,9 @@ impl ExecutionBlockHeader { nonce: Hash64::zero(), base_fee_per_gas: payload.base_fee_per_gas(), withdrawals_root: rlp_withdrawals_root, - data_gas_used: rlp_data_gas_used, - excess_data_gas: rlp_excess_data_gas, + blob_gas_used: rlp_blob_gas_used, + excess_blob_gas: rlp_excess_blob_gas, + parent_beacon_block_root: rlp_parent_beacon_block_root, } } } diff --git a/consensus/types/src/execution_payload.rs b/consensus/types/src/execution_payload.rs index 4186cdf198e..41241b21bee 100644 --- a/consensus/types/src/execution_payload.rs +++ b/consensus/types/src/execution_payload.rs @@ -83,14 +83,12 @@ pub struct ExecutionPayload { pub transactions: Transactions, #[superstruct(only(Capella, Deneb))] pub withdrawals: Withdrawals, - #[superstruct(only(Deneb))] + #[superstruct(only(Deneb), partial_getter(copy))] #[serde(with = "serde_utils::quoted_u64")] - #[superstruct(getter(copy))] - pub data_gas_used: u64, - #[superstruct(only(Deneb))] + pub blob_gas_used: u64, + #[superstruct(only(Deneb), partial_getter(copy))] #[serde(with = "serde_utils::quoted_u64")] - #[superstruct(getter(copy))] - pub excess_data_gas: u64, + pub excess_blob_gas: u64, } impl<'a, T: EthSpec> ExecutionPayloadRef<'a, T> { diff --git a/consensus/types/src/execution_payload_header.rs b/consensus/types/src/execution_payload_header.rs index 5cb89f74420..9f68ca940b2 100644 --- a/consensus/types/src/execution_payload_header.rs +++ b/consensus/types/src/execution_payload_header.rs @@ -80,11 +80,11 @@ pub struct ExecutionPayloadHeader { #[superstruct(only(Deneb))] #[serde(with = "serde_utils::quoted_u64")] #[superstruct(getter(copy))] - pub data_gas_used: u64, + pub blob_gas_used: u64, #[superstruct(only(Deneb))] #[serde(with = "serde_utils::quoted_u64")] #[superstruct(getter(copy))] - pub excess_data_gas: u64, + pub excess_blob_gas: u64, } impl ExecutionPayloadHeader { @@ -155,8 +155,8 @@ impl ExecutionPayloadHeaderCapella { block_hash: self.block_hash, transactions_root: self.transactions_root, withdrawals_root: self.withdrawals_root, - data_gas_used: 0, - excess_data_gas: 0, + blob_gas_used: 0, + excess_blob_gas: 0, } } } @@ -221,8 +221,8 @@ impl<'a, T: EthSpec> From<&'a ExecutionPayloadDeneb> for ExecutionPayloadHead block_hash: payload.block_hash, transactions_root: payload.transactions.tree_hash_root(), withdrawals_root: payload.withdrawals.tree_hash_root(), - data_gas_used: payload.data_gas_used, - excess_data_gas: payload.excess_data_gas, + blob_gas_used: payload.blob_gas_used, + excess_blob_gas: payload.excess_blob_gas, } } } diff --git a/consensus/types/src/voluntary_exit.rs b/consensus/types/src/voluntary_exit.rs index 02686fef9ad..446029b560b 100644 --- a/consensus/types/src/voluntary_exit.rs +++ b/consensus/types/src/voluntary_exit.rs @@ -1,5 +1,5 @@ use crate::{ - test_utils::TestRandom, ChainSpec, Domain, Epoch, Fork, Hash256, SecretKey, SignedRoot, + test_utils::TestRandom, ChainSpec, Domain, Epoch, ForkName, Hash256, SecretKey, SignedRoot, SignedVoluntaryExit, }; @@ -37,16 +37,20 @@ impl VoluntaryExit { pub fn sign( self, secret_key: &SecretKey, - fork: &Fork, genesis_validators_root: Hash256, spec: &ChainSpec, ) -> SignedVoluntaryExit { - let domain = spec.get_domain( - self.epoch, - Domain::VoluntaryExit, - fork, - genesis_validators_root, - ); + let fork_name = spec.fork_name_at_epoch(self.epoch); + let fork_version = match fork_name { + ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + spec.fork_version_for_name(fork_name) + } + // EIP-7044 + ForkName::Deneb => spec.fork_version_for_name(ForkName::Capella), + }; + let domain = + spec.compute_domain(Domain::VoluntaryExit, fork_version, genesis_validators_root); + let message = self.signing_root(domain); SignedVoluntaryExit { message: self, diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index 56812552599..65e5cc7be54 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -734,6 +734,38 @@ fn builder_fallback_flags() { ); }, ); + run_payload_builder_flag_test_with_config( + "builder", + "http://meow.cats", + Some("ignore-builder-override-suggestion-threshold"), + Some("53.4"), + |config| { + assert_eq!( + config + .execution_layer + .as_ref() + .unwrap() + .ignore_builder_override_suggestion_threshold, + 53.4f32 + ); + }, + ); + run_payload_builder_flag_test_with_config( + "builder", + "http://meow.cats", + None, + None, + |config| { + assert_eq!( + config + .execution_layer + .as_ref() + .unwrap() + .ignore_builder_override_suggestion_threshold, + 10.0f32 + ); + }, + ); } #[test] diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile index f568c87cf5a..a1a71e0b2af 100644 --- a/testing/ef_tests/Makefile +++ b/testing/ef_tests/Makefile @@ -1,4 +1,4 @@ -TESTS_TAG := v1.4.0-alpha.2 +TESTS_TAG := v1.4.0-beta.1 TESTS = general minimal mainnet TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS)) diff --git a/testing/ef_tests/src/cases/operations.rs b/testing/ef_tests/src/cases/operations.rs index e823f6273ca..4ccd9e8a710 100644 --- a/testing/ef_tests/src/cases/operations.rs +++ b/testing/ef_tests/src/cases/operations.rs @@ -11,7 +11,7 @@ use state_processing::{ errors::BlockProcessingError, process_block_header, process_execution_payload, process_operations::{ - altair, base, process_attester_slashings, process_bls_to_execution_changes, + altair_deneb, base, process_attester_slashings, process_bls_to_execution_changes, process_deposits, process_exits, process_proposer_slashings, }, process_sync_aggregate, process_withdrawals, VerifyBlockRoot, VerifySignatures, @@ -103,7 +103,14 @@ impl Operation for Attestation { | BeaconState::Capella(_) | BeaconState::Deneb(_) => { initialize_progressive_balances_cache(state, None, spec)?; - altair::process_attestation(state, self, 0, &mut ctxt, VerifySignatures::True, spec) + altair_deneb::process_attestation( + state, + self, + 0, + &mut ctxt, + VerifySignatures::True, + spec, + ) } } } diff --git a/testing/execution_engine_integration/src/test_rig.rs b/testing/execution_engine_integration/src/test_rig.rs index 59630172dbc..2aaff30f577 100644 --- a/testing/execution_engine_integration/src/test_rig.rs +++ b/testing/execution_engine_integration/src/test_rig.rs @@ -270,7 +270,13 @@ impl TestRig { head_root, proposer_index, // TODO: think about how to test different forks - PayloadAttributes::new(timestamp, prev_randao, Address::repeat_byte(42), None), + PayloadAttributes::new( + timestamp, + prev_randao, + Address::repeat_byte(42), + None, + None, + ), ) .await; @@ -309,7 +315,7 @@ impl TestRig { .get_suggested_fee_recipient(proposer_index) .await; let payload_attributes = - PayloadAttributes::new(timestamp, prev_randao, suggested_fee_recipient, None); + PayloadAttributes::new(timestamp, prev_randao, suggested_fee_recipient, None, None); let valid_payload = self .ee_a .execution_layer @@ -358,10 +364,11 @@ impl TestRig { * Provide the valid payload back to the EE again. */ + // TODO: again consider forks here let status = self .ee_a .execution_layer - .notify_new_payload(&valid_payload, None) + .notify_new_payload(valid_payload.clone().try_into().unwrap()) .await .unwrap(); assert_eq!(status, PayloadStatus::Valid); @@ -409,12 +416,13 @@ impl TestRig { * Provide an invalidated payload to the EE. */ + // TODO: again think about forks here let mut invalid_payload = valid_payload.clone(); *invalid_payload.prev_randao_mut() = Hash256::from_low_u64_be(42); let status = self .ee_a .execution_layer - .notify_new_payload(&invalid_payload, None) + .notify_new_payload(invalid_payload.try_into().unwrap()) .await .unwrap(); assert!(matches!( @@ -449,7 +457,7 @@ impl TestRig { .get_suggested_fee_recipient(proposer_index) .await; let payload_attributes = - PayloadAttributes::new(timestamp, prev_randao, suggested_fee_recipient, None); + PayloadAttributes::new(timestamp, prev_randao, suggested_fee_recipient, None, None); let second_payload = self .ee_a .execution_layer @@ -473,10 +481,11 @@ impl TestRig { * Provide the second payload back to the EE again. */ + // TODO: again consider forks here let status = self .ee_a .execution_layer - .notify_new_payload(&second_payload, None) + .notify_new_payload(second_payload.clone().try_into().unwrap()) .await .unwrap(); assert_eq!(status, PayloadStatus::Valid); @@ -493,7 +502,7 @@ impl TestRig { // To save sending proposer preparation data, just set the fee recipient // to the fee recipient configured for EE A. let payload_attributes = - PayloadAttributes::new(timestamp, prev_randao, Address::repeat_byte(42), None); + PayloadAttributes::new(timestamp, prev_randao, Address::repeat_byte(42), None, None); let slot = Slot::new(42); let head_block_root = Hash256::repeat_byte(100); let validator_index = 0; @@ -520,10 +529,11 @@ impl TestRig { * * Provide the second payload, without providing the first. */ + // TODO: again consider forks here let status = self .ee_b .execution_layer - .notify_new_payload(&second_payload, None) + .notify_new_payload(second_payload.clone().try_into().unwrap()) .await .unwrap(); // TODO: we should remove the `Accepted` status here once Geth fixes it @@ -561,10 +571,11 @@ impl TestRig { * Provide the first payload to the EE. */ + // TODO: again consider forks here let status = self .ee_b .execution_layer - .notify_new_payload(&valid_payload, None) + .notify_new_payload(valid_payload.clone().try_into().unwrap()) .await .unwrap(); assert_eq!(status, PayloadStatus::Valid); @@ -578,7 +589,7 @@ impl TestRig { let status = self .ee_b .execution_layer - .notify_new_payload(&second_payload, None) + .notify_new_payload(second_payload.clone().try_into().unwrap()) .await .unwrap(); assert_eq!(status, PayloadStatus::Valid); diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index 903a3cc56d5..9492ad588f9 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -21,12 +21,12 @@ use task_executor::TaskExecutor; use types::{ attestation::Error as AttestationError, graffiti::GraffitiString, AbstractExecPayload, Address, AggregateAndProof, Attestation, BeaconBlock, BlindedPayload, BlobSidecarList, ChainSpec, - ContributionAndProof, Domain, Epoch, EthSpec, Fork, Graffiti, Hash256, Keypair, PublicKeyBytes, - SelectionProof, Signature, SignedAggregateAndProof, SignedBeaconBlock, SignedBlobSidecar, - SignedBlobSidecarList, SignedContributionAndProof, SignedRoot, SignedValidatorRegistrationData, - SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, SyncCommitteeContribution, - SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData, - VoluntaryExit, + ContributionAndProof, Domain, Epoch, EthSpec, Fork, ForkName, Graffiti, Hash256, Keypair, + PublicKeyBytes, SelectionProof, Signature, SignedAggregateAndProof, SignedBeaconBlock, + SignedBlobSidecar, SignedBlobSidecarList, SignedContributionAndProof, SignedRoot, + SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, + SyncCommitteeContribution, SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, + ValidatorRegistrationData, VoluntaryExit, }; use validator_dir::ValidatorDir; @@ -371,11 +371,35 @@ impl ValidatorStore { } fn signing_context(&self, domain: Domain, signing_epoch: Epoch) -> SigningContext { - SigningContext { - domain, - epoch: signing_epoch, - fork: self.fork(signing_epoch), - genesis_validators_root: self.genesis_validators_root, + if domain == Domain::VoluntaryExit { + match self.spec.fork_name_at_epoch(signing_epoch) { + ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + SigningContext { + domain, + epoch: signing_epoch, + fork: self.fork(signing_epoch), + genesis_validators_root: self.genesis_validators_root, + } + } + // EIP-7044 + ForkName::Deneb => SigningContext { + domain, + epoch: signing_epoch, + fork: Fork { + previous_version: self.spec.capella_fork_version, + current_version: self.spec.capella_fork_version, + epoch: signing_epoch, + }, + genesis_validators_root: self.genesis_validators_root, + }, + } + } else { + SigningContext { + domain, + epoch: signing_epoch, + fork: self.fork(signing_epoch), + genesis_validators_root: self.genesis_validators_root, + } } }