From 675a231b45efd7f1755e7a7cf1312e2cdb5c1fd5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Thu, 8 Feb 2024 02:00:49 +0000 Subject: [PATCH 01/24] increment peer per topic count on graft messages (#5212) * increment peer per topic count on graft messages --- beacon_node/lighthouse_network/src/gossipsub/behaviour.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/beacon_node/lighthouse_network/src/gossipsub/behaviour.rs b/beacon_node/lighthouse_network/src/gossipsub/behaviour.rs index 8a89e5904a1..9769adca278 100644 --- a/beacon_node/lighthouse_network/src/gossipsub/behaviour.rs +++ b/beacon_node/lighthouse_network/src/gossipsub/behaviour.rs @@ -1380,7 +1380,11 @@ where tracing::error!(peer_id = %peer_id, "Peer non-existent when handling graft"); return; }; - connected_peer.topics.insert(topic.clone()); + if connected_peer.topics.insert(topic.clone()) { + if let Some(m) = self.metrics.as_mut() { + m.inc_topic_peers(topic); + } + } } // we don't GRAFT to/from explicit peers; complain loudly if this happens From 4db84de563eda826d4a7abc4d1f4ce8a7e043569 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 8 Feb 2024 02:40:47 +0000 Subject: [PATCH 02/24] Improve network parameters (#5177) * Modify network parameters for current mainnet conditions --- beacon_node/lighthouse_network/src/config.rs | 2 +- beacon_node/lighthouse_network/src/discovery/mod.rs | 2 +- beacon_node/lighthouse_network/src/service/mod.rs | 2 +- beacon_node/src/config.rs | 2 -- 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/beacon_node/lighthouse_network/src/config.rs b/beacon_node/lighthouse_network/src/config.rs index 93ab3d2d81b..5b13730f971 100644 --- a/beacon_node/lighthouse_network/src/config.rs +++ b/beacon_node/lighthouse_network/src/config.rs @@ -350,7 +350,7 @@ impl Default for Config { enr_udp6_port: None, enr_quic6_port: None, enr_tcp6_port: None, - target_peers: 50, + target_peers: 100, gs_config, discv5_config, boot_nodes_enr: vec![], diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index 829124e1233..6659ba1d26f 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -59,7 +59,7 @@ const MAX_DISCOVERY_RETRY: usize = 3; /// Note: we always allow a single FindPeers query, so we would be /// running a maximum of `MAX_CONCURRENT_SUBNET_QUERIES + 1` /// discovery queries at a time. -const MAX_CONCURRENT_SUBNET_QUERIES: usize = 2; +const MAX_CONCURRENT_SUBNET_QUERIES: usize = 4; /// The max number of subnets to search for in a single subnet discovery query. const MAX_SUBNETS_IN_QUERY: usize = 3; /// The number of closest peers to search for when doing a regular peer search. diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index a66670e7faf..879f13ad7c8 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -51,7 +51,7 @@ mod gossip_cache; pub mod gossipsub_scoring_parameters; pub mod utils; /// The number of peers we target per subnet for discovery queries. -pub const TARGET_SUBNET_PEERS: usize = 6; +pub const TARGET_SUBNET_PEERS: usize = 3; const MAX_IDENTIFY_ADDRESSES: usize = 10; diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index b485775dbfe..1699c51784f 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -1122,8 +1122,6 @@ pub fn set_network_config( config.target_peers = target_peers_str .parse::() .map_err(|_| format!("Invalid number of target peers: {}", target_peers_str))?; - } else { - config.target_peers = 80; // default value } if let Some(value) = cli_args.value_of("network-load") { From 0b59d10ab6150c6d2ecb4fceb76ca7ef77d5c217 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Thu, 8 Feb 2024 08:10:51 +0530 Subject: [PATCH 03/24] Fix backfill stalling (#5192) * Prevent early short circuit in `peer_disconnected` * lint --- beacon_node/network/src/sync/backfill_sync/mod.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/beacon_node/network/src/sync/backfill_sync/mod.rs b/beacon_node/network/src/sync/backfill_sync/mod.rs index 0d7e7c16c36..4e24aca07ff 100644 --- a/beacon_node/network/src/sync/backfill_sync/mod.rs +++ b/beacon_node/network/src/sync/backfill_sync/mod.rs @@ -332,7 +332,16 @@ impl BackFillSync { } // If we have run out of peers in which to retry this batch, the backfill state // transitions to a paused state. - self.retry_batch_download(network, id)?; + // We still need to reset the state for all the affected batches, so we should not + // short circuit early + if self.retry_batch_download(network, id).is_err() { + debug!( + self.log, + "Batch could not be retried"; + "batch_id" => id, + "error" => "no synced peers" + ); + } } else { debug!(self.log, "Batch not found while removing peer"; "peer" => %peer_id, "batch" => id) From 0c3fef59b3c6697ee84e4f89a8537c097eacfc5a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Thu, 8 Feb 2024 02:40:54 +0000 Subject: [PATCH 04/24] improve libp2p connected peer metrics (#4870) * improve libp2p connected peer metrics * separate discv5 port from libp2p for NAT open * use metric family for DISCOVERY_BYTES * Merge branch 'unstable' of https://github.com/sigp/lighthouse into improve-metrics --- beacon_node/http_api/src/lib.rs | 10 +-- .../lighthouse_network/src/discovery/mod.rs | 8 ++- beacon_node/lighthouse_network/src/metrics.rs | 68 +++---------------- .../src/peer_manager/mod.rs | 18 ----- .../src/peer_manager/network_behaviour.rs | 26 ++++--- common/lighthouse_metrics/src/lib.rs | 10 +++ common/system_health/src/lib.rs | 25 +++++-- 7 files changed, 65 insertions(+), 100 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index f89bc3c29fb..ad841b20e3a 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -67,7 +67,7 @@ use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; use sysinfo::{System, SystemExt}; -use system_health::observe_system_health_bn; +use system_health::{observe_nat, observe_system_health_bn}; use task_spawner::{Priority, TaskSpawner}; use tokio::sync::{ mpsc::{Sender, UnboundedSender}, @@ -4071,13 +4071,7 @@ pub fn serve( .and(warp::path::end()) .then(|task_spawner: TaskSpawner| { task_spawner.blocking_json_task(Priority::P1, move || { - Ok(api_types::GenericResponse::from( - lighthouse_network::metrics::NAT_OPEN - .as_ref() - .map(|v| v.get()) - .unwrap_or(0) - != 0, - )) + Ok(api_types::GenericResponse::from(observe_nat())) }) }); diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index 6659ba1d26f..c1781c85832 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -1003,11 +1003,13 @@ impl NetworkBehaviour for Discovery { } discv5::Event::SocketUpdated(socket_addr) => { info!(self.log, "Address updated"; "ip" => %socket_addr.ip(), "udp_port" => %socket_addr.port()); - metrics::inc_counter(&metrics::ADDRESS_UPDATE_COUNT); - metrics::check_nat(); + // We have SOCKET_UPDATED messages. This occurs when discovery has a majority of + // users reporting an external port and our ENR gets updated. + // Which means we are able to do NAT traversal. + metrics::set_gauge_vec(&metrics::NAT_OPEN, &["discv5"], 1); + // Discv5 will have updated our local ENR. We save the updated version // to disk. - if (self.update_ports.tcp4 && socket_addr.is_ipv4()) || (self.update_ports.tcp6 && socket_addr.is_ipv6()) { diff --git a/beacon_node/lighthouse_network/src/metrics.rs b/beacon_node/lighthouse_network/src/metrics.rs index ae02b689d81..ada48d6ebd0 100644 --- a/beacon_node/lighthouse_network/src/metrics.rs +++ b/beacon_node/lighthouse_network/src/metrics.rs @@ -1,29 +1,17 @@ pub use lighthouse_metrics::*; lazy_static! { - pub static ref NAT_OPEN: Result = try_create_int_counter( + pub static ref NAT_OPEN: Result = try_create_int_gauge_vec( "nat_open", - "An estimate indicating if the local node is exposed to the internet." + "An estimate indicating if the local node is reachable from external nodes", + &["protocol"] ); pub static ref ADDRESS_UPDATE_COUNT: Result = try_create_int_counter( "libp2p_address_update_total", "Count of libp2p socked updated events (when our view of our IP address has changed)" ); - pub static ref PEERS_CONNECTED: Result = try_create_int_gauge( - "libp2p_peers", - "Count of libp2p peers currently connected" - ); - - pub static ref TCP_PEERS_CONNECTED: Result = try_create_int_gauge( - "libp2p_tcp_peers", - "Count of libp2p peers currently connected via TCP" - ); - - pub static ref QUIC_PEERS_CONNECTED: Result = try_create_int_gauge( - "libp2p_quic_peers", - "Count of libp2p peers currently connected via QUIC" - ); - + pub static ref PEERS_CONNECTED: Result = + try_create_int_gauge_vec("libp2p_peers", "Count of libp2p peers currently connected", &["direction", "transport"]); pub static ref PEER_CONNECT_EVENT_COUNT: Result = try_create_int_counter( "libp2p_peer_connect_event_total", "Count of libp2p peer connect events (not the current number of connected peers)" @@ -32,13 +20,10 @@ lazy_static! { "libp2p_peer_disconnect_event_total", "Count of libp2p peer disconnect events" ); - pub static ref DISCOVERY_SENT_BYTES: Result = try_create_int_gauge( - "discovery_sent_bytes", - "The number of bytes sent in discovery" - ); - pub static ref DISCOVERY_RECV_BYTES: Result = try_create_int_gauge( - "discovery_recv_bytes", - "The number of bytes received in discovery" + pub static ref DISCOVERY_BYTES: Result = try_create_int_gauge_vec( + "discovery_bytes", + "The number of bytes sent and received in discovery", + &["direction"] ); pub static ref DISCOVERY_QUEUE: Result = try_create_int_gauge( "discovery_queue_size", @@ -135,17 +120,6 @@ lazy_static! { &["type"] ); - /* - * Inbound/Outbound peers - */ - /// The number of peers that dialed us. - pub static ref NETWORK_INBOUND_PEERS: Result = - try_create_int_gauge("network_inbound_peers","The number of peers that are currently connected that have dialed us."); - - /// The number of peers that we dialed us. - pub static ref NETWORK_OUTBOUND_PEERS: Result = - try_create_int_gauge("network_outbound_peers","The number of peers that are currently connected that we dialed."); - /* * Peer Reporting */ @@ -156,31 +130,11 @@ lazy_static! { ); } -/// Checks if we consider the NAT open. -/// -/// Conditions for an open NAT: -/// 1. We have 1 or more SOCKET_UPDATED messages. This occurs when discovery has a majority of -/// users reporting an external port and our ENR gets updated. -/// 2. We have 0 SOCKET_UPDATED messages (can be true if the port was correct on boot), then we -/// rely on whether we have any inbound messages. If we have no socket update messages, but -/// manage to get at least one inbound peer, we are exposed correctly. -pub fn check_nat() { - // NAT is already deemed open. - if NAT_OPEN.as_ref().map(|v| v.get()).unwrap_or(0) != 0 { - return; - } - if ADDRESS_UPDATE_COUNT.as_ref().map(|v| v.get()).unwrap_or(0) != 0 - || NETWORK_INBOUND_PEERS.as_ref().map(|v| v.get()).unwrap_or(0) != 0_i64 - { - inc_counter(&NAT_OPEN); - } -} - pub fn scrape_discovery_metrics() { let metrics = discv5::metrics::Metrics::from(discv5::Discv5::::raw_metrics()); set_float_gauge(&DISCOVERY_REQS, metrics.unsolicited_requests_per_second); set_gauge(&DISCOVERY_SESSIONS, metrics.active_sessions as i64); - set_gauge(&DISCOVERY_SENT_BYTES, metrics.bytes_sent as i64); - set_gauge(&DISCOVERY_RECV_BYTES, metrics.bytes_recv as i64); + set_gauge_vec(&DISCOVERY_BYTES, &["inbound"], metrics.bytes_recv as i64); + set_gauge_vec(&DISCOVERY_BYTES, &["outbound"], metrics.bytes_sent as i64); } diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index e4976a0d374..8227c3420b2 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -726,29 +726,14 @@ impl PeerManager { return; } - let mut connected_peer_count = 0; - let mut inbound_connected_peers = 0; - let mut outbound_connected_peers = 0; let mut clients_per_peer = HashMap::new(); for (_peer, peer_info) in self.network_globals.peers.read().connected_peers() { - connected_peer_count += 1; - if let PeerConnectionStatus::Connected { n_in, .. } = peer_info.connection_status() { - if *n_in > 0 { - inbound_connected_peers += 1; - } else { - outbound_connected_peers += 1; - } - } *clients_per_peer .entry(peer_info.client().kind.to_string()) .or_default() += 1; } - metrics::set_gauge(&metrics::PEERS_CONNECTED, connected_peer_count); - metrics::set_gauge(&metrics::NETWORK_INBOUND_PEERS, inbound_connected_peers); - metrics::set_gauge(&metrics::NETWORK_OUTBOUND_PEERS, outbound_connected_peers); - for client_kind in ClientKind::iter() { let value = clients_per_peer.get(&client_kind.to_string()).unwrap_or(&0); metrics::set_gauge_vec( @@ -853,11 +838,8 @@ impl PeerManager { // start a ping and status timer for the peer self.status_peers.insert(*peer_id); - let connected_peers = self.network_globals.connected_peers() as i64; - // increment prometheus metrics metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT); - metrics::set_gauge(&metrics::PEERS_CONNECTED, connected_peers); true } diff --git a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs index cb60906f632..1c9274af233 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs @@ -154,6 +154,8 @@ impl NetworkBehaviour for PeerManager { self.on_dial_failure(peer_id); } FromSwarm::ExternalAddrConfirmed(_) => { + // We have an external address confirmed, means we are able to do NAT traversal. + metrics::set_gauge_vec(&metrics::NAT_OPEN, &["libp2p"], 1); // TODO: we likely want to check this against our assumed external tcp // address } @@ -243,14 +245,14 @@ impl PeerManager { self.events.push(PeerManagerEvent::MetaData(peer_id)); } - // Check NAT if metrics are enabled - if self.network_globals.local_enr.read().udp4().is_some() { - metrics::check_nat(); - } - // increment prometheus metrics if self.metrics_enabled { let remote_addr = endpoint.get_remote_address(); + let direction = if endpoint.is_dialer() { + "outbound" + } else { + "inbound" + }; match remote_addr.iter().find(|proto| { matches!( proto, @@ -258,10 +260,10 @@ impl PeerManager { ) }) { Some(multiaddr::Protocol::QuicV1) => { - metrics::inc_gauge(&metrics::QUIC_PEERS_CONNECTED); + metrics::inc_gauge_vec(&metrics::PEERS_CONNECTED, &[direction, "quic"]); } Some(multiaddr::Protocol::Tcp(_)) => { - metrics::inc_gauge(&metrics::TCP_PEERS_CONNECTED); + metrics::inc_gauge_vec(&metrics::PEERS_CONNECTED, &[direction, "tcp"]); } Some(_) => unreachable!(), None => { @@ -339,6 +341,12 @@ impl PeerManager { let remote_addr = endpoint.get_remote_address(); // Update the prometheus metrics if self.metrics_enabled { + let direction = if endpoint.is_dialer() { + "outbound" + } else { + "inbound" + }; + match remote_addr.iter().find(|proto| { matches!( proto, @@ -346,10 +354,10 @@ impl PeerManager { ) }) { Some(multiaddr::Protocol::QuicV1) => { - metrics::dec_gauge(&metrics::QUIC_PEERS_CONNECTED); + metrics::dec_gauge_vec(&metrics::PEERS_CONNECTED, &[direction, "quic"]); } Some(multiaddr::Protocol::Tcp(_)) => { - metrics::dec_gauge(&metrics::TCP_PEERS_CONNECTED); + metrics::dec_gauge_vec(&metrics::PEERS_CONNECTED, &[direction, "tcp"]); } // If it's an unknown protocol we already logged when connection was established. _ => {} diff --git a/common/lighthouse_metrics/src/lib.rs b/common/lighthouse_metrics/src/lib.rs index 5d25bb313f6..ba9900cc507 100644 --- a/common/lighthouse_metrics/src/lib.rs +++ b/common/lighthouse_metrics/src/lib.rs @@ -244,6 +244,16 @@ pub fn inc_counter_vec(int_counter_vec: &Result, name: &[&str]) { } } +/// Sets the `int_counter_vec` with the given `name` to the `amount`, +/// should only be called with `ammount`s equal or above the current value +/// as per Prometheus spec, the `counter` type should only go up. +pub fn set_counter_vec_by(int_counter_vec: &Result, name: &[&str], amount: u64) { + if let Some(counter) = get_int_counter(int_counter_vec, name) { + counter.reset(); + counter.inc_by(amount); + } +} + pub fn inc_counter_vec_by(int_counter_vec: &Result, name: &[&str], amount: u64) { if let Some(counter) = get_int_counter(int_counter_vec, name) { counter.inc_by(amount); diff --git a/common/system_health/src/lib.rs b/common/system_health/src/lib.rs index d10540e506c..ec64ce31ad3 100644 --- a/common/system_health/src/lib.rs +++ b/common/system_health/src/lib.rs @@ -198,6 +198,25 @@ pub fn observe_system_health_vc( } } +/// Observes if NAT traversal is possible. +pub fn observe_nat() -> bool { + let discv5_nat = lighthouse_network::metrics::get_int_gauge( + &lighthouse_network::metrics::NAT_OPEN, + &["discv5"], + ) + .map(|g| g.get() == 1) + .unwrap_or_default(); + + let libp2p_nat = lighthouse_network::metrics::get_int_gauge( + &lighthouse_network::metrics::NAT_OPEN, + &["libp2p"], + ) + .map(|g| g.get() == 1) + .unwrap_or_default(); + + discv5_nat && libp2p_nat +} + /// Observes the Beacon Node system health. pub fn observe_system_health_bn( sysinfo: Arc>, @@ -223,11 +242,7 @@ pub fn observe_system_health_bn( .unwrap_or_else(|| (String::from("None"), 0, 0)); // Determine if the NAT is open or not. - let nat_open = lighthouse_network::metrics::NAT_OPEN - .as_ref() - .map(|v| v.get()) - .unwrap_or(0) - != 0; + let nat_open = observe_nat(); SystemHealthBN { system_health, From e4705967150bd589baa7cd8cd43a6ab8f330f039 Mon Sep 17 00:00:00 2001 From: zilayo <84344709+zilayo@users.noreply.github.com> Date: Thu, 8 Feb 2024 02:40:58 +0000 Subject: [PATCH 05/24] chore(docs): amend port guidance to enable QUIC support (#5029) * chore(docs): amend port guidance to enable QUIC support --- book/src/advanced_networking.md | 57 ++++++++++++++++++++++++-------- book/src/docker.md | 8 ++--- book/src/faq.md | 58 +++++++++++++++++---------------- 3 files changed, 77 insertions(+), 46 deletions(-) diff --git a/book/src/advanced_networking.md b/book/src/advanced_networking.md index b1f05450c48..5fabf57d568 100644 --- a/book/src/advanced_networking.md +++ b/book/src/advanced_networking.md @@ -40,7 +40,7 @@ drastically and use the (recommended) default. ### NAT Traversal (Port Forwarding) -Lighthouse, by default, uses port 9000 for both TCP and UDP. Lighthouse will +Lighthouse, by default, uses port 9000 for both TCP and UDP. Since v4.5.0, Lighthouse will also attempt to make QUIC connections via UDP port 9001 by default. Lighthouse will still function if it is behind a NAT without any port mappings. Although Lighthouse still functions, we recommend that some mechanism is used to ensure that your Lighthouse node is publicly accessible. This will typically improve @@ -50,8 +50,8 @@ peers for your node and overall improve the Ethereum consensus network. Lighthouse currently supports UPnP. If UPnP is enabled on your router, Lighthouse will automatically establish the port mappings for you (the beacon node will inform you of established routes in this case). If UPnP is not -enabled, we recommend you to manually set up port mappings to both of Lighthouse's -TCP and UDP ports (9000 by default). +enabled, we recommend you to manually set up port mappings to Lighthouse's +TCP and UDP ports (9000 TCP/UDP, and 9001 UDP by default). > Note: Lighthouse needs to advertise its publicly accessible ports in > order to inform its peers that it is contactable and how to connect to it. @@ -66,7 +66,7 @@ TCP and UDP ports (9000 by default). The steps to do port forwarding depends on the router, but the general steps are given below: 1. Determine the default gateway IP: -- On Linux: open a terminal and run `ip route | grep default`, the result should look something similar to `default via 192.168.50.1 dev wlp2s0 proto dhcp metric 600`. The `192.168.50.1` is your router management default gateway IP. +- On Linux: open a terminal and run `ip route | grep default`, the result should look something similar to `default via 192.168.50.1 dev wlp2s0 proto dhcp metric 600`. The `192.168.50.1` is your router management default gateway IP. - On MacOS: open a terminal and run `netstat -nr|grep default` and it should return the default gateway IP. - On Windows: open a command prompt and run `ipconfig` and look for the `Default Gateway` which will show you the gateway IP. @@ -74,16 +74,22 @@ The steps to do port forwarding depends on the router, but the general steps are 2. Login to the router management page. The login credentials are usually available in the manual or the router, or it can be found on a sticker underneath the router. You can also try the login credentials for some common router brands listed [here](https://www.noip.com/support/knowledgebase/general-port-forwarding-guide/). -3. Navigate to the port forward settings in your router. The exact step depends on the router, but typically it will fall under the "Advanced" section, under the name "port forwarding" or "virtual server". +3. Navigate to the port forward settings in your router. The exact step depends on the router, but typically it will fall under the "Advanced" section, under the name "port forwarding" or "virtual server". 4. Configure a port forwarding rule as below: - Protocol: select `TCP/UDP` or `BOTH` - External port: `9000` - Internal port: `9000` -- IP address: Usually there is a dropdown list for you to select the device. Choose the device that is running Lighthouse +- IP address: Usually there is a dropdown list for you to select the device. Choose the device that is running Lighthouse. -5. To check that you have successfully open the ports, go to [yougetsignal](https://www.yougetsignal.com/tools/open-ports/) and enter `9000` in the `port number`. If it shows "open", then you have successfully set up port forwarding. If it shows "closed", double check your settings, and also check that you have allowed firewall rules on port 9000. +Since V4.5.0 port 9001/UDP is also used for QUIC support. +- Protocol: select `UDP` +- External port: `9001` +- Internal port: `9001` +- IP address: Choose the device that is running Lighthouse. + +5. To check that you have successfully opened the ports, go to [yougetsignal](https://www.yougetsignal.com/tools/open-ports/) and enter `9000` in the `port number`. If it shows "open", then you have successfully set up port forwarding. If it shows "closed", double check your settings, and also check that you have allowed firewall rules on port 9000. Note: this will only confirm if port 9000/TCP is open. You will need to ensure you have correctly setup port forwarding for the UDP ports (`9000` and `9001` by default). ### ENR Configuration @@ -125,6 +131,9 @@ IPv4 only: TCP and UDP. - `--listen-address :: --port 9909 --discovery-port 9999` will listen over IPv6 using port `9909` for TCP and port `9999` for UDP. +- By default, QUIC listens for UDP connections using a port number that is one greater than the specified port. + If the specified port is 9909, QUIC will use port 9910 for IPv6 UDP connections. + This can be configured with `--quic-port`. To listen over both IPv4 and IPv6: - Set two listening addresses using the `--listen-address` flag twice ensuring @@ -133,18 +142,38 @@ To listen over both IPv4 and IPv6: that this behaviour differs from the Ipv6 only case described above. - If necessary, set the `--port6` flag to configure the port used for TCP and UDP over IPv6. This flag has no effect when listening over IPv6 only. -- If necessary, set the `--discovery-port6` flag to configure the IPv6 UDP +- If necessary, set the `--discovery-port6` flag to configure the IPv6 UDP port. This will default to the value given to `--port6` if not set. This flag has no effect when listening over IPv6 only. +- If necessary, set the `--quic-port6` flag to configure the port used by QUIC for + UDP over IPv6. This will default to the value given to `--port6` + 1. This flag + has no effect when listening over IPv6 only. ##### Configuration Examples -- `--listen-address :: --listen-address 0.0.0.0 --port 9909` will listen - over IPv4 using port `9909` for TCP and UDP. It will also listen over IPv6 but - using the default value for `--port6` for UDP and TCP (`9090`). -- `--listen-address :: --listen-address --port 9909 --discovery-port6 9999` - will have the same configuration as before except for the IPv6 UDP socket, - which will use port `9999`. +> When using `--listen-address :: --listen-address 0.0.0.0 --port 9909`, listening will be set up as follows: +> +> **IPv4**: +> +> It listens on port `9909` for both TCP and UDP. +> QUIC will use the next sequential port `9910` for UDP. +> +> **IPv6**: +> +> It listens on the default value of --port6 (`9090`) for both UDP and TCP. +> QUIC will use port `9091` for UDP, which is the default `--port6` value (`9090`) + 1. + +> When using `--listen-address :: --listen-address --port 9909 --discovery-port6 9999`, listening will be set up as follows: +> +> **IPv4**: +> +> It listens on port `9909` for both TCP and UDP. +> QUIC will use the next sequential port `9910` for UDP. +> +> **IPv6**: +> +> It listens on the default value of `--port6` (`9090`) for TCP, and port `9999` for UDP. +> QUIC will use port `9091` for UDP, which is the default `--port6` value (`9090`) + 1. #### Configuring Lighthouse to advertise IPv6 reachable addresses Lighthouse supports IPv6 to connect to other nodes both over IPv6 exclusively, diff --git a/book/src/docker.md b/book/src/docker.md index defa89517e1..c48c745a044 100644 --- a/book/src/docker.md +++ b/book/src/docker.md @@ -112,7 +112,7 @@ docker run lighthouse:local lighthouse --help You can run a Docker beacon node with the following command: ```bash -docker run -p 9000:9000/tcp -p 9000:9000/udp -p 127.0.0.1:5052:5052 -v $HOME/.lighthouse:/root/.lighthouse sigp/lighthouse lighthouse --network mainnet beacon --http --http-address 0.0.0.0 +docker run -p 9000:9000/tcp -p 9000:9000/udp -p 9001:9001/udp -p 127.0.0.1:5052:5052 -v $HOME/.lighthouse:/root/.lighthouse sigp/lighthouse lighthouse --network mainnet beacon --http --http-address 0.0.0.0 ``` > To join the Goerli testnet, use `--network goerli` instead. @@ -135,18 +135,18 @@ docker run -v $HOME/.lighthouse:/root/.lighthouse sigp/lighthouse lighthouse bea ### Ports -In order to be a good peer and serve other peers you should expose port `9000` for both TCP and UDP. +In order to be a good peer and serve other peers you should expose port `9000` for both TCP and UDP, and port `9001` for UDP. Use the `-p` flag to do this: ```bash -docker run -p 9000:9000/tcp -p 9000:9000/udp sigp/lighthouse lighthouse beacon +docker run -p 9000:9000/tcp -p 9000:9000/udp -p 9001:9001/udp sigp/lighthouse lighthouse beacon ``` If you use the `--http` flag you may also want to expose the HTTP port with `-p 127.0.0.1:5052:5052`. ```bash -docker run -p 9000:9000/tcp -p 9000:9000/udp -p 127.0.0.1:5052:5052 sigp/lighthouse lighthouse beacon --http --http-address 0.0.0.0 +docker run -p 9000:9000/tcp -p 9000:9000/udp -p 9001:9001/udp -p 127.0.0.1:5052:5052 sigp/lighthouse lighthouse beacon --http --http-address 0.0.0.0 ``` [docker_hub]: https://hub.docker.com/repository/docker/sigp/lighthouse/ diff --git a/book/src/faq.md b/book/src/faq.md index c1717f7fc88..b8b267f17c6 100644 --- a/book/src/faq.md +++ b/book/src/faq.md @@ -80,13 +80,13 @@ The `WARN Execution engine called failed` log is shown when the beacon node cann `error: Reqwest(reqwest::Error { kind: Request, url: Url { scheme: "http", cannot_be_a_base: false, username: "", password: None, host: Some(Ipv4(127.0.0.1)), port: Some(8551), path: "/", query: None, fragment: None }, source: TimedOut }), service: exec` which says `TimedOut` at the end of the message. This means that the execution engine has not responded in time to the beacon node. One option is to add the flags `--execution-timeout-multiplier 3` and `--disable-lock-timeouts` to the beacon node. However, if the error persists, it is worth digging further to find out the cause. There are a few reasons why this can occur: -1. The execution engine is not synced. Check the log of the execution engine to make sure that it is synced. If it is syncing, wait until it is synced and the error will disappear. You will see the beacon node logs `INFO Execution engine online` when it is synced. +1. The execution engine is not synced. Check the log of the execution engine to make sure that it is synced. If it is syncing, wait until it is synced and the error will disappear. You will see the beacon node logs `INFO Execution engine online` when it is synced. 1. The computer is overloaded. Check the CPU and RAM usage to see if it has overloaded. You can use `htop` to check for CPU and RAM usage. 1. Your SSD is slow. Check if your SSD is in "The Bad" list [here](https://gist.github.com/yorickdowne/f3a3e79a573bf35767cd002cc977b038). If your SSD is in "The Bad" list, it means it cannot keep in sync to the network and you may want to consider upgrading to a better SSD. If the reason for the error message is caused by no. 1 above, you may want to look further. If the execution engine is out of sync suddenly, it is usually caused by ungraceful shutdown. The common causes for ungraceful shutdown are: -- Power outage. If power outages are an issue at your place, consider getting a UPS to avoid ungraceful shutdown of services. -- The service file is not stopped properly. To overcome this, make sure that the process is stopped properly, e.g., during client updates. +- Power outage. If power outages are an issue at your place, consider getting a UPS to avoid ungraceful shutdown of services. +- The service file is not stopped properly. To overcome this, make sure that the process is stopped properly, e.g., during client updates. - Out of memory (oom) error. This can happen when the system memory usage has reached its maximum and causes the execution engine to be killed. When this occurs, the log file will show `Main process exited, code=killed, status=9/KILL`. You can also run `sudo journalctl -a --since "18 hours ago" | grep -i "killed process` to confirm that the execution client has been killed due to oom. If you are using geth as the execution client, a short term solution is to reduce the resources used. For example, you can reduce the cache by adding the flag `--cache 2048`. If the oom occurs rather frequently, a long term solution is to increase the memory capacity of the computer. ### My beacon node is stuck at downloading historical block using checkpoint sync. What should I do? @@ -98,8 +98,8 @@ INFO Downloading historical blocks est_time: --, distance: 4524545 slo ``` If the same log appears every minute and you do not see progress in downloading historical blocks, you can try one of the followings: - - - Check the number of peers you are connected to. If you have low peers (less than 50), try to do port forwarding on the port 9000 TCP/UDP to increase peer count. + + - Check the number of peers you are connected to. If you have low peers (less than 50), try to do port forwarding on the ports 9000 TCP/UDP and 9001 UDP to increase peer count. - Restart the beacon node. @@ -110,7 +110,7 @@ INFO Block from HTTP API already known` WARN Could not publish message error: Duplicate, service: libp2p ``` -This error usually happens when users are running mev-boost. The relay will publish the block on the network before returning it back to you. After the relay published the block on the network, it will propagate through nodes, and it happens quite often that your node will receive the block from your connected peers via gossip first, before getting the block from the relay, hence the message `duplicate`. +This error usually happens when users are running mev-boost. The relay will publish the block on the network before returning it back to you. After the relay published the block on the network, it will propagate through nodes, and it happens quite often that your node will receive the block from your connected peers via gossip first, before getting the block from the relay, hence the message `duplicate`. In short, it is nothing to worry about. @@ -124,7 +124,7 @@ WARN Head is optimistic execution_block_hash: 0x47e7555f1d4215d1ad409b1ac1 It means the beacon node will follow the chain, but it will not be able to attest or produce blocks. This is because the execution client is not synced, so the beacon chain cannot verify the authenticity of the chain head, hence the word `optimistic`. What you need to do is to make sure that the execution client is up and syncing. Once the execution client is synced, the error will disappear. -### My beacon node logs `CRIT Beacon block processing error error: ValidatorPubkeyCacheLockTimeout, service: beacon`, what should I do? +### My beacon node logs `CRIT Beacon block processing error error: ValidatorPubkeyCacheLockTimeout, service: beacon`, what should I do? An example of the log is shown below: @@ -133,7 +133,7 @@ CRIT Beacon block processing error error: ValidatorPubkeyCacheLockTime WARN BlockProcessingFailure outcome: ValidatorPubkeyCacheLockTimeout, msg: unexpected condition in processing block. ``` -A `Timeout` error suggests that the computer may be overloaded at the moment, for example, the execution client is still syncing. You may use the flag `--disable-lock-timeouts` to silence this error, although it will not fix the underlying slowness. Nevertheless, this is a relatively harmless log, and the error should go away once the resources used are back to normal. +A `Timeout` error suggests that the computer may be overloaded at the moment, for example, the execution client is still syncing. You may use the flag `--disable-lock-timeouts` to silence this error, although it will not fix the underlying slowness. Nevertheless, this is a relatively harmless log, and the error should go away once the resources used are back to normal. ### My beacon node logs `WARN BlockProcessingFailure outcome: MissingBeaconBlock`, what should I do? @@ -143,7 +143,7 @@ An example of the full log is shown below: WARN BlockProcessingFailure outcome: MissingBeaconBlock(0xbdba211f8d72029554e405d8e4906690dca807d1d7b1bc8c9b88d7970f1648bc), msg: unexpected condition in processing block. ``` -`MissingBeaconBlock` suggests that the database has corrupted. You should wipe the database and use [Checkpoint Sync](./checkpoint-sync.md) to resync the beacon chain. +`MissingBeaconBlock` suggests that the database has corrupted. You should wipe the database and use [Checkpoint Sync](./checkpoint-sync.md) to resync the beacon chain. ### After checkpoint sync, the progress of `downloading historical blocks` is slow. Why? @@ -173,7 +173,7 @@ The error is `503 Service Unavailable`. This means that the beacon node is still ERRO Failed to download attester duties err: FailedToDownloadAttesters("Some endpoints failed, num_failed: 2 http://localhost:5052/ => Unavailable(NotSynced), http://localhost:5052/ => RequestFailed(ServerMessage(ErrorMessage { code: 503, message: \"SERVICE_UNAVAILABLE: beacon node is syncing ``` -This means that the validator client is sending requests to the beacon node. However, as the beacon node is still syncing, it is therefore unable to fulfil the request. The error will disappear once the beacon node is synced. +This means that the validator client is sending requests to the beacon node. However, as the beacon node is still syncing, it is therefore unable to fulfil the request. The error will disappear once the beacon node is synced. ### My beacon node logs `WARN Error signalling fork choice waiter`, what should I do? @@ -268,9 +268,9 @@ repeats until the queue is cleared. The churn limit is summarised in the table b
-| Number of active validators | Validators activated per epoch | Validators activated per day | +| Number of active validators | Validators activated per epoch | Validators activated per day | |-------------------|--------------------------------------------|----| -| 327679 or less | 4 | 900 | +| 327679 or less | 4 | 900 | | 327680-393215 | 5 | 1125 | | 393216-458751 | 6 | 1350 | 458752-524287 | 7 | 1575 @@ -285,7 +285,7 @@ repeats until the queue is cleared. The churn limit is summarised in the table b
-For example, the number of active validators on Mainnet is about 574000 on May 2023. This means that 8 validators can be activated per epoch or 1800 per day (it is noted that the same applies to the exit queue). If, for example, there are 9000 validators waiting to be activated, this means that the waiting time can take up to 5 days. +For example, the number of active validators on Mainnet is about 574000 on May 2023. This means that 8 validators can be activated per epoch or 1800 per day (it is noted that the same applies to the exit queue). If, for example, there are 9000 validators waiting to be activated, this means that the waiting time can take up to 5 days. Once a validator has been activated, congratulations! It's time to produce blocks and attestations! @@ -298,14 +298,14 @@ duplicate your JSON keystores and don't run `lighthouse vc` twice). This will le However, there are some components which can be configured with redundancy. See the [Redundancy](./redundancy.md) guide for more information. -### I am missing attestations. Why? +### I am missing attestations. Why? The first thing is to ensure both consensus and execution clients are synced with the network. If they are synced, there may still be some issues with the node setup itself that is causing the missed attestations. Check the setup to ensure that: - the clock is synced - the computer has sufficient resources and is not overloaded - the internet is working well - you have sufficient peers -You can see more information on the [Ethstaker KB](https://ethstaker.gitbook.io/ethstaker-knowledge-base/help/missed-attestations). +You can see more information on the [Ethstaker KB](https://ethstaker.gitbook.io/ethstaker-knowledge-base/help/missed-attestations). Another cause for missing attestations is delays during block processing. When this happens, the debug logs will show (debug logs can be found under `$datadir/beacon/logs`): @@ -313,14 +313,14 @@ Another cause for missing attestations is delays during block processing. When t DEBG Delayed head block set_as_head_delay: Some(93.579425ms), imported_delay: Some(1.460405278s), observed_delay: Some(2.540811921s), block_delay: 4.094796624s, slot: 6837344, proposer_index: 211108, block_root: 0x2c52231c0a5a117401f5231585de8aa5dd963bc7cbc00c544e681342eedd1700, service: beacon ``` -The fields to look for are `imported_delay > 1s` and `observed_delay < 3s`. The `imported_delay` is how long the node took to process the block. The `imported_delay` of larger than 1 second suggests that there is slowness in processing the block. It could be due to high CPU usage, high I/O disk usage or the clients are doing some background maintenance processes. The `observed_delay` is determined mostly by the proposer and partly by your networking setup (e.g., how long it took for the node to receive the block). The `observed_delay` of less than 3 seconds means that the block is not arriving late from the block proposer. Combining the above, this implies that the validator should have been able to attest to the block, but failed due to slowness in the node processing the block. +The fields to look for are `imported_delay > 1s` and `observed_delay < 3s`. The `imported_delay` is how long the node took to process the block. The `imported_delay` of larger than 1 second suggests that there is slowness in processing the block. It could be due to high CPU usage, high I/O disk usage or the clients are doing some background maintenance processes. The `observed_delay` is determined mostly by the proposer and partly by your networking setup (e.g., how long it took for the node to receive the block). The `observed_delay` of less than 3 seconds means that the block is not arriving late from the block proposer. Combining the above, this implies that the validator should have been able to attest to the block, but failed due to slowness in the node processing the block. ### Sometimes I miss the attestation head vote, resulting in penalty. Is this normal? In general, it is unavoidable to have some penalties occasionally. This is particularly the case when you are assigned to attest on the first slot of an epoch and if the proposer of that slot releases the block late, then you will get penalised for missing the target and head votes. Your attestation performance does not only depend on your own setup, but also on everyone elses performance. -You could also check for the sync aggregate participation percentage on block explorers such as [beaconcha.in](https://beaconcha.in/). A low sync aggregate participation percentage (e.g., 60-70%) indicates that the block that you are assigned to attest to may be published late. As a result, your validator fails to correctly attest to the block. +You could also check for the sync aggregate participation percentage on block explorers such as [beaconcha.in](https://beaconcha.in/). A low sync aggregate participation percentage (e.g., 60-70%) indicates that the block that you are assigned to attest to may be published late. As a result, your validator fails to correctly attest to the block. Another possible reason for missing the head vote is due to a chain "reorg". A reorg can happen if the proposer publishes block `n` late, and the proposer of block `n+1` builds upon block `n-1` instead of `n`. This is called a "reorg". Due to the reorg, block `n` was never included in the chain. If you are assigned to attest at slot `n`, it is possible you may still attest to block `n` despite most of the network recognizing the block as being late. In this case you will miss the head reward. @@ -388,7 +388,7 @@ If the ports are open, you should have incoming peers. To check that you have in If you have incoming peers, it should return a lot of data containing information of peers. If the response is empty, it means that you have no incoming peers and there the ports are not open. You may want to double check if the port forward was correctly set up. -2. Check that you do not lower the number of peers using the flag `--target-peers`. The default is 80. A lower value set will lower the maximum number of peers your node can connect to, which may potentially interrupt the validator performance. We recommend users to leave the `--target peers` untouched to keep a diverse set of peers. +2. Check that you do not lower the number of peers using the flag `--target-peers`. The default is 80. A lower value set will lower the maximum number of peers your node can connect to, which may potentially interrupt the validator performance. We recommend users to leave the `--target peers` untouched to keep a diverse set of peers. 3. Ensure that you have a quality router for the internet connection. For example, if you connect the router to many devices including the node, it may be possible that the router cannot handle all routing tasks, hence struggling to keep up the number of peers. Therefore, using a quality router for the node is important to keep a healthy number of peers. @@ -435,8 +435,8 @@ For these reasons, we recommend that you make your node publicly accessible. Lighthouse supports UPnP. If you are behind a NAT with a router that supports UPnP, you can simply ensure UPnP is enabled (Lighthouse will inform you in its initial logs if a route has been established). You can also manually [set up port mappings/port forwarding](./advanced_networking.md#how-to-open-ports) in your router to your local Lighthouse instance. By default, -Lighthouse uses port 9000 for both TCP and UDP. Opening both these ports will -make your Lighthouse node maximally contactable. +Lighthouse uses port 9000 for both TCP and UDP, and optionally 9001 UDP for QUIC support. +Opening these ports will make your Lighthouse node maximally contactable. ### How can I monitor my validators? @@ -449,7 +449,7 @@ Monitoring](./validator-monitoring.md) for more information. Lighthouse has also The setting on the beacon node is the same for both cases below. In the beacon node, specify `lighthouse bn --http-address local_IP` so that the beacon node is listening on the local network rather than `localhost`. You can find the `local_IP` by running the command `hostname -I | awk '{print $1}'` on the server running the beacon node. 1. If the beacon node and validator clients are on different servers *in the same network*, the setting in the validator client is as follows: - + Use the flag `--beacon-nodes` to point to the beacon node. For example, `lighthouse vc --beacon-nodes http://local_IP:5052` where `local_IP` is the local IP address of the beacon node and `5052` is the default `http-port` of the beacon node. If you have firewall setup, e.g., `ufw`, you will need to allow port 5052 (assuming that the default port is used) with `sudo ufw allow 5052`. Note: this will allow all IP addresses to access the HTTP API of the beacon node. If you are on an untrusted network (e.g., a university or public WiFi) or the host is exposed to the internet, use apply IP-address filtering as described later in this section. @@ -472,7 +472,7 @@ The setting on the beacon node is the same for both cases below. In the beacon n If you have firewall setup, e.g., `ufw`, you will need to allow connections to port 5052 (assuming that the default port is used). Since the beacon node HTTP/HTTPS API is public-facing (i.e., the 5052 port is now exposed to the internet due to port forwarding), we strongly recommend users to apply IP-address filtering to the BN/VC connection from malicious actors. This can be done using the command: - + ``` sudo ufw allow from vc_IP_address proto tcp to any port 5052 ``` @@ -485,14 +485,16 @@ It is also worth noting that the `--beacon-nodes` flag can also be used for redu No. Lighthouse will auto-detect the change and update your Ethereum Node Record (ENR). You just need to make sure you are not manually setting the ENR with `--enr-address` (which, for common use cases, this flag is not used). ### How to change the TCP/UDP port 9000 that Lighthouse listens on? -Use the flag ```--port ``` in the beacon node. This flag can be useful when you are running two beacon nodes at the same time. You can leave one beacon node as the default port 9000, and configure the second beacon node to listen on, e.g., ```--port 9001```. +Use the flag `--port ` in the beacon node. This flag can be useful when you are running two beacon nodes at the same time. You can leave one beacon node as the default port 9000, and configure the second beacon node to listen on, e.g., `--port 9100`. +Since V4.5.0, Lighthouse supports QUIC and by default will use the value of `--port` + 1 to listen via UDP (default `9001`). +This can be configured by using the flag `--quic-port`. Refer to [Advanced Networking](./advanced_networking.md#nat-traversal-port-forwarding) for more information. ### Lighthouse `v4.3.0` introduces a change where a node will subscribe to only 2 subnets in total. I am worried that this will impact my validators return. -Previously, having more validators means subscribing to more subnets. Since the change, a node will now only subscribe to 2 subnets in total. This will bring about significant reductions in bandwidth for nodes with multiple validators. +Previously, having more validators means subscribing to more subnets. Since the change, a node will now only subscribe to 2 subnets in total. This will bring about significant reductions in bandwidth for nodes with multiple validators. + +While subscribing to more subnets can ensure you have peers on a wider range of subnets, these subscriptions consume resources and bandwidth. This does not significantly increase the performance of the node, however it does benefit other nodes on the network. -While subscribing to more subnets can ensure you have peers on a wider range of subnets, these subscriptions consume resources and bandwidth. This does not significantly increase the performance of the node, however it does benefit other nodes on the network. - If you would still like to subscribe to all subnets, you can use the flag `subscribe-all-subnets`. This may improve the block rewards by 1-5%, though it comes at the cost of a much higher bandwidth requirement. ### How to know how many of my peers are connected via QUIC? @@ -549,7 +551,7 @@ which says that the version is v4.1.0. ### Does Lighthouse have pruning function like the execution client to save disk space? -There is no pruning of Lighthouse database for now. However, since v4.2.0, a feature to only sync back to the weak subjectivity point (approximately 5 months) when syncing via a checkpoint sync was added. This will help to save disk space since the previous behaviour will sync back to the genesis by default. +There is no pruning of Lighthouse database for now. However, since v4.2.0, a feature to only sync back to the weak subjectivity point (approximately 5 months) when syncing via a checkpoint sync was added. This will help to save disk space since the previous behaviour will sync back to the genesis by default. ### Can I use a HDD for the freezer database and only have the hot db on SSD? @@ -557,7 +559,7 @@ Yes, you can do so by using the flag `--freezer-dir /path/to/freezer_db` in the ### Can Lighthouse log in local timestamp instead of UTC? -The reason why Lighthouse logs in UTC is due to the dependency on an upstream library that is [yet to be resolved](https://github.com/sigp/lighthouse/issues/3130). Alternatively, using the flag `disable-log-timestamp` in combination with systemd will suppress the UTC timestamps and print the logs in local timestamps. +The reason why Lighthouse logs in UTC is due to the dependency on an upstream library that is [yet to be resolved](https://github.com/sigp/lighthouse/issues/3130). Alternatively, using the flag `disable-log-timestamp` in combination with systemd will suppress the UTC timestamps and print the logs in local timestamps. ### My hard disk is full and my validator is down. What should I do? From 6f442f2bb8a6ea976943b5fd0c620d04d25f8245 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 8 Feb 2024 21:05:08 +1100 Subject: [PATCH 06/24] Improve database compaction and `prune-states` (#5142) * Fix no-op state prune check * Compact freezer DB after pruning * Refine DB compaction * Add blobs-db options to inspect/compact * Better key size * Fix compaction end key --- beacon_node/store/src/hot_cold_store.rs | 3 + beacon_node/store/src/leveldb_store.rs | 28 ++---- beacon_node/store/src/lib.rs | 18 +++- beacon_node/store/src/memory_store.rs | 2 +- database_manager/src/lib.rs | 117 ++++++++++++++++++++---- 5 files changed, 127 insertions(+), 41 deletions(-) diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 49bc75393c1..4bdb0deca33 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -2376,6 +2376,9 @@ impl, Cold: ItemStore> HotColdDB self.cold_db.do_atomically(cold_ops)?; } + // In order to reclaim space, we need to compact the freezer DB as well. + self.cold_db.compact()?; + Ok(()) } } diff --git a/beacon_node/store/src/leveldb_store.rs b/beacon_node/store/src/leveldb_store.rs index 62619dd2c74..d799bdedd3b 100644 --- a/beacon_node/store/src/leveldb_store.rs +++ b/beacon_node/store/src/leveldb_store.rs @@ -154,25 +154,15 @@ impl KeyValueStore for LevelDB { self.transaction_mutex.lock() } - /// Compact all values in the states and states flag columns. - fn compact(&self) -> Result<(), Error> { - let endpoints = |column: DBColumn| { - ( - BytesKey::from_vec(get_key_for_col(column.as_str(), Hash256::zero().as_bytes())), - BytesKey::from_vec(get_key_for_col( - column.as_str(), - Hash256::repeat_byte(0xff).as_bytes(), - )), - ) - }; - - for (start_key, end_key) in [ - endpoints(DBColumn::BeaconStateTemporary), - endpoints(DBColumn::BeaconState), - endpoints(DBColumn::BeaconStateSummary), - ] { - self.db.compact(&start_key, &end_key); - } + fn compact_column(&self, column: DBColumn) -> Result<(), Error> { + // Use key-size-agnostic keys [] and 0xff..ff with a minimum of 32 bytes to account for + // columns that may change size between sub-databases or schema versions. + let start_key = BytesKey::from_vec(get_key_for_col(column.as_str(), &[])); + let end_key = BytesKey::from_vec(get_key_for_col( + column.as_str(), + &vec![0xff; std::cmp::max(column.key_size(), 32)], + )); + self.db.compact(&start_key, &end_key); Ok(()) } diff --git a/beacon_node/store/src/lib.rs b/beacon_node/store/src/lib.rs index eacd28d2db6..e86689b0cf1 100644 --- a/beacon_node/store/src/lib.rs +++ b/beacon_node/store/src/lib.rs @@ -80,8 +80,22 @@ pub trait KeyValueStore: Sync + Send + Sized + 'static { /// this method. In future we may implement a safer mandatory locking scheme. fn begin_rw_transaction(&self) -> MutexGuard<()>; - /// Compact the database, freeing space used by deleted items. - fn compact(&self) -> Result<(), Error>; + /// Compact a single column in the database, freeing space used by deleted items. + fn compact_column(&self, column: DBColumn) -> Result<(), Error>; + + /// Compact a default set of columns that are likely to free substantial space. + fn compact(&self) -> Result<(), Error> { + // Compact state and block related columns as they are likely to have the most churn, + // i.e. entries being created and deleted. + for column in [ + DBColumn::BeaconState, + DBColumn::BeaconStateSummary, + DBColumn::BeaconBlock, + ] { + self.compact_column(column)?; + } + Ok(()) + } /// Iterate through all keys and values in a particular column. fn iter_column(&self, column: DBColumn) -> ColumnIter { diff --git a/beacon_node/store/src/memory_store.rs b/beacon_node/store/src/memory_store.rs index c2e494dce6a..302d2c2add2 100644 --- a/beacon_node/store/src/memory_store.rs +++ b/beacon_node/store/src/memory_store.rs @@ -108,7 +108,7 @@ impl KeyValueStore for MemoryStore { self.transaction_mutex.lock() } - fn compact(&self) -> Result<(), Error> { + fn compact_column(&self, _column: DBColumn) -> Result<(), Error> { Ok(()) } } diff --git a/database_manager/src/lib.rs b/database_manager/src/lib.rs index 87a1a8bd149..3583d9e2792 100644 --- a/database_manager/src/lib.rs +++ b/database_manager/src/lib.rs @@ -77,7 +77,15 @@ pub fn inspect_cli_app<'a, 'b>() -> App<'a, 'b> { Arg::with_name("freezer") .long("freezer") .help("Inspect the freezer DB rather than the hot DB") - .takes_value(false), + .takes_value(false) + .conflicts_with("blobs-db"), + ) + .arg( + Arg::with_name("blobs-db") + .long("blobs-db") + .help("Inspect the blobs DB rather than the hot DB") + .takes_value(false) + .conflicts_with("freezer"), ) .arg( Arg::with_name("output-dir") @@ -88,6 +96,34 @@ pub fn inspect_cli_app<'a, 'b>() -> App<'a, 'b> { ) } +pub fn compact_cli_app<'a, 'b>() -> App<'a, 'b> { + App::new("compact") + .setting(clap::AppSettings::ColoredHelp) + .about("Compact database manually") + .arg( + Arg::with_name("column") + .long("column") + .value_name("TAG") + .help("3-byte column ID (see `DBColumn`)") + .takes_value(true) + .required(true), + ) + .arg( + Arg::with_name("freezer") + .long("freezer") + .help("Inspect the freezer DB rather than the hot DB") + .takes_value(false) + .conflicts_with("blobs-db"), + ) + .arg( + Arg::with_name("blobs-db") + .long("blobs-db") + .help("Inspect the blobs DB rather than the hot DB") + .takes_value(false) + .conflicts_with("freezer"), + ) +} + pub fn prune_payloads_app<'a, 'b>() -> App<'a, 'b> { App::new("prune-payloads") .alias("prune_payloads") @@ -162,6 +198,7 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .subcommand(migrate_cli_app()) .subcommand(version_cli_app()) .subcommand(inspect_cli_app()) + .subcommand(compact_cli_app()) .subcommand(prune_payloads_app()) .subcommand(prune_blobs_app()) .subcommand(prune_states_app()) @@ -251,6 +288,7 @@ pub struct InspectConfig { skip: Option, limit: Option, freezer: bool, + blobs_db: bool, /// Configures where the inspect output should be stored. output_dir: PathBuf, } @@ -261,6 +299,7 @@ fn parse_inspect_config(cli_args: &ArgMatches) -> Result let skip = clap_utils::parse_optional(cli_args, "skip")?; let limit = clap_utils::parse_optional(cli_args, "limit")?; let freezer = cli_args.is_present("freezer"); + let blobs_db = cli_args.is_present("blobs-db"); let output_dir: PathBuf = clap_utils::parse_optional(cli_args, "output-dir")?.unwrap_or_else(PathBuf::new); @@ -270,6 +309,7 @@ fn parse_inspect_config(cli_args: &ArgMatches) -> Result skip, limit, freezer, + blobs_db, output_dir, }) } @@ -277,32 +317,20 @@ fn parse_inspect_config(cli_args: &ArgMatches) -> Result pub fn inspect_db( inspect_config: InspectConfig, client_config: ClientConfig, - runtime_context: &RuntimeContext, - log: Logger, ) -> Result<(), String> { - let spec = runtime_context.eth2_config.spec.clone(); let hot_path = client_config.get_db_path(); let cold_path = client_config.get_freezer_db_path(); let blobs_path = client_config.get_blobs_db_path(); - let db = HotColdDB::, LevelDB>::open( - &hot_path, - &cold_path, - &blobs_path, - |_, _, _| Ok(()), - client_config.store, - spec, - log, - ) - .map_err(|e| format!("{:?}", e))?; - let mut total = 0; let mut num_keys = 0; let sub_db = if inspect_config.freezer { - &db.cold_db + LevelDB::::open(&cold_path).map_err(|e| format!("Unable to open freezer DB: {e:?}"))? + } else if inspect_config.blobs_db { + LevelDB::::open(&blobs_path).map_err(|e| format!("Unable to open blobs DB: {e:?}"))? } else { - &db.hot_db + LevelDB::::open(&hot_path).map_err(|e| format!("Unable to open hot DB: {e:?}"))? }; let skip = inspect_config.skip.unwrap_or(0); @@ -385,6 +413,50 @@ pub fn inspect_db( Ok(()) } +pub struct CompactConfig { + column: DBColumn, + freezer: bool, + blobs_db: bool, +} + +fn parse_compact_config(cli_args: &ArgMatches) -> Result { + let column = clap_utils::parse_required(cli_args, "column")?; + let freezer = cli_args.is_present("freezer"); + let blobs_db = cli_args.is_present("blobs-db"); + Ok(CompactConfig { + column, + freezer, + blobs_db, + }) +} + +pub fn compact_db( + compact_config: CompactConfig, + client_config: ClientConfig, + log: Logger, +) -> Result<(), Error> { + let hot_path = client_config.get_db_path(); + let cold_path = client_config.get_freezer_db_path(); + let blobs_path = client_config.get_blobs_db_path(); + let column = compact_config.column; + + let (sub_db, db_name) = if compact_config.freezer { + (LevelDB::::open(&cold_path)?, "freezer_db") + } else if compact_config.blobs_db { + (LevelDB::::open(&blobs_path)?, "blobs_db") + } else { + (LevelDB::::open(&hot_path)?, "hot_db") + }; + info!( + log, + "Compacting database"; + "db" => db_name, + "column" => ?column + ); + sub_db.compact_column(column)?; + Ok(()) +} + pub struct MigrateConfig { to: SchemaVersion, } @@ -538,7 +610,10 @@ pub fn prune_states( // Check that the user has confirmed they want to proceed. if !prune_config.confirm { match db.get_anchor_info() { - Some(anchor_info) if anchor_info.state_upper_limit == STATE_UPPER_LIMIT_NO_RETAIN => { + Some(anchor_info) + if anchor_info.state_lower_limit == 0 + && anchor_info.state_upper_limit == STATE_UPPER_LIMIT_NO_RETAIN => + { info!(log, "States have already been pruned"); return Ok(()); } @@ -586,7 +661,11 @@ pub fn run(cli_args: &ArgMatches<'_>, env: Environment) -> Result } ("inspect", Some(cli_args)) => { let inspect_config = parse_inspect_config(cli_args)?; - inspect_db(inspect_config, client_config, &context, log) + inspect_db::(inspect_config, client_config) + } + ("compact", Some(cli_args)) => { + let compact_config = parse_compact_config(cli_args)?; + compact_db::(compact_config, client_config, log).map_err(format_err) } ("prune-payloads", Some(_)) => { prune_payloads(client_config, &context, log).map_err(format_err) From 4b19eac8cef6c93feb0c86363cf06c4b683a4478 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Fri, 9 Feb 2024 03:57:19 +1100 Subject: [PATCH 07/24] Remove `curve25519-dalek` patch (#5214) * Remove patch dependencies --- Cargo.lock | 8 +++++--- Cargo.toml | 3 --- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4fc8ef8fe97..1082c2fd449 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1583,8 +1583,9 @@ dependencies = [ [[package]] name = "curve25519-dalek" -version = "4.1.1" -source = "git+https://github.com/jimmygchen/curve25519-dalek.git?rev=24019783e9bb9dc1464e7e503732f273a69969c6#24019783e9bb9dc1464e7e503732f273a69969c6" +version = "4.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a677b8922c94e01bdbb12126b0bc852f00447528dee1782229af9c720c3f348" dependencies = [ "cfg-if", "cpufeatures", @@ -1600,7 +1601,8 @@ dependencies = [ [[package]] name = "curve25519-dalek-derive" version = "0.1.1" -source = "git+https://github.com/jimmygchen/curve25519-dalek.git?rev=24019783e9bb9dc1464e7e503732f273a69969c6#24019783e9bb9dc1464e7e503732f273a69969c6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index ce3b47012c8..12a7474212a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -235,6 +235,3 @@ inherits = "release" lto = "fat" codegen-units = 1 incremental = false - -[patch.crates-io] -curve25519-dalek = { git = "https://github.com/jimmygchen/curve25519-dalek.git", rev = "24019783e9bb9dc1464e7e503732f273a69969c6" } From e3533584845462de3b2b6e41476fd3d41f2ae053 Mon Sep 17 00:00:00 2001 From: Akihito Nakano Date: Fri, 9 Feb 2024 01:57:26 +0900 Subject: [PATCH 08/24] Update to warp v0.3.6 (#5172) * Update to warp 0.3.6 * Update Cargo.lock --- Cargo.lock | 88 ++++++++++++------------------------------------------ Cargo.toml | 3 +- 2 files changed, 20 insertions(+), 71 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1082c2fd449..83673d239c0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3112,7 +3112,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35bd3cf68c183738046838e300353e4716c674dc5e56890de4826801a6622a28" dependencies = [ "futures-io", - "rustls 0.21.10", + "rustls", ] [[package]] @@ -3735,7 +3735,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.5.5", + "socket2 0.4.10", "tokio", "tower-service", "tracing", @@ -3770,9 +3770,9 @@ dependencies = [ "futures-util", "http 0.2.11", "hyper 0.14.28", - "rustls 0.21.10", + "rustls", "tokio", - "tokio-rustls 0.24.1", + "tokio-rustls", ] [[package]] @@ -4624,7 +4624,7 @@ dependencies = [ "quinn", "rand", "ring 0.16.20", - "rustls 0.21.10", + "rustls", "socket2 0.5.5", "thiserror", "tokio", @@ -4695,8 +4695,8 @@ dependencies = [ "libp2p-identity", "rcgen", "ring 0.16.20", - "rustls 0.21.10", - "rustls-webpki 0.101.7", + "rustls", + "rustls-webpki", "thiserror", "x509-parser", "yasna", @@ -6486,7 +6486,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash", - "rustls 0.21.10", + "rustls", "thiserror", "tokio", "tracing", @@ -6502,7 +6502,7 @@ dependencies = [ "rand", "ring 0.16.20", "rustc-hash", - "rustls 0.21.10", + "rustls", "slab", "thiserror", "tinyvec", @@ -6733,15 +6733,15 @@ dependencies = [ "once_cell", "percent-encoding", "pin-project-lite", - "rustls 0.21.10", - "rustls-pemfile 1.0.4", + "rustls", + "rustls-pemfile", "serde", "serde_json", "serde_urlencoded", "system-configuration", "tokio", "tokio-native-tls", - "tokio-rustls 0.24.1", + "tokio-rustls", "tokio-util 0.7.10", "tower-service", "url", @@ -6964,24 +6964,10 @@ checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", "ring 0.17.7", - "rustls-webpki 0.101.7", + "rustls-webpki", "sct", ] -[[package]] -name = "rustls" -version = "0.22.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e87c9956bd9807afa1f77e0f7594af32566e830e088a5576d27c5b6f30f49d41" -dependencies = [ - "log", - "ring 0.17.7", - "rustls-pki-types", - "rustls-webpki 0.102.1", - "subtle", - "zeroize", -] - [[package]] name = "rustls-pemfile" version = "1.0.4" @@ -6991,22 +6977,6 @@ dependencies = [ "base64 0.21.7", ] -[[package]] -name = "rustls-pemfile" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35e4980fa29e4c4b212ffb3db068a564cbf560e51d3944b7c88bd8bf5bec64f4" -dependencies = [ - "base64 0.21.7", - "rustls-pki-types", -] - -[[package]] -name = "rustls-pki-types" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e9d979b3ce68192e42760c7810125eb6cf2ea10efae545a156063e61f314e2a" - [[package]] name = "rustls-webpki" version = "0.101.7" @@ -7017,17 +6987,6 @@ dependencies = [ "untrusted 0.9.0", ] -[[package]] -name = "rustls-webpki" -version = "0.102.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef4ca26037c909dedb327b48c3327d0ba91d3dd3c4e05dad328f210ffb68e95b" -dependencies = [ - "ring 0.17.7", - "rustls-pki-types", - "untrusted 0.9.0", -] - [[package]] name = "rustversion" version = "1.0.14" @@ -8334,18 +8293,7 @@ version = "0.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" dependencies = [ - "rustls 0.21.10", - "tokio", -] - -[[package]] -name = "tokio-rustls" -version = "0.25.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "775e0c0f0adb3a2f22a00c4745d728b479985fc15ee7ca6a2608388c5569860f" -dependencies = [ - "rustls 0.22.2", - "rustls-pki-types", + "rustls", "tokio", ] @@ -8991,7 +8939,8 @@ dependencies = [ [[package]] name = "warp" version = "0.3.6" -source = "git+https://github.com/seanmonstar/warp.git#7b07043cee0ca24e912155db4e8f6d9ab7c049ed" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1e92e22e03ff1230c03a1a8ee37d2f89cd489e2e541b7550d6afad96faed169" dependencies = [ "bytes", "futures-channel", @@ -9004,13 +8953,14 @@ dependencies = [ "mime_guess", "percent-encoding", "pin-project", - "rustls-pemfile 2.0.0", + "rustls-pemfile", "scoped-tls", "serde", "serde_json", "serde_urlencoded", "tokio", - "tokio-rustls 0.25.0", + "tokio-rustls", + "tokio-stream", "tokio-util 0.7.10", "tower-service", "tracing", diff --git a/Cargo.toml b/Cargo.toml index 12a7474212a..a7f44551ee8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -170,8 +170,7 @@ tree_hash = "0.5" tree_hash_derive = "0.5" url = "2" uuid = { version = "0.8", features = ["serde", "v4"] } -# TODO update to warp 0.3.6 after released. -warp = { git = "https://github.com/seanmonstar/warp.git", default-features = false, features = ["tls"] } +warp = { version = "0.3.6", default-features = false, features = ["tls"] } zeroize = { version = "1", features = ["zeroize_derive"] } zip = "0.6" From 4172d9f75c052f963d440ee2dad00f3512542117 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 8 Feb 2024 13:08:21 -0500 Subject: [PATCH 09/24] Update to consensus spec v1.4.0-beta.6 (#5094) * get latest ef tests passing * fix tests * Fix invalid payload recovery tests * Merge branch 'unstable' into update-to-spec-v1.4.0-beta.6 * Revert "fix tests" This reverts commit 0c875b02e032ebd9bebd822183a1c3d10d333a4c. * Fix fork choice def. tests * Update beacon_node/beacon_chain/tests/payload_invalidation.rs --- .../tests/payload_invalidation.rs | 107 ++++++++++-------- .../ffg_updates.rs | 53 +++++++-- .../fork_choice_test_definition/no_votes.rs | 14 ++- consensus/proto_array/src/proto_array.rs | 13 +-- testing/ef_tests/Makefile | 2 +- testing/ef_tests/check_all_files_accessed.py | 3 +- testing/ef_tests/tests/tests.rs | 3 +- 7 files changed, 119 insertions(+), 76 deletions(-) diff --git a/beacon_node/beacon_chain/tests/payload_invalidation.rs b/beacon_node/beacon_chain/tests/payload_invalidation.rs index a0b7fbd365a..597d53fddd2 100644 --- a/beacon_node/beacon_chain/tests/payload_invalidation.rs +++ b/beacon_node/beacon_chain/tests/payload_invalidation.rs @@ -1820,81 +1820,94 @@ struct InvalidHeadSetup { } impl InvalidHeadSetup { + /// This function aims to produce two things: + /// + /// 1. A chain where the only viable head block has an invalid execution payload. + /// 2. A block (`fork_block`) which will become the head of the chain when + /// it is imported. async fn new() -> InvalidHeadSetup { + let slots_per_epoch = E::slots_per_epoch(); let mut rig = InvalidPayloadRig::new().enable_attestations(); rig.move_to_terminal_block(); rig.import_block(Payload::Valid).await; // Import a valid transition block. - // Import blocks until the first time the chain finalizes. + // Import blocks until the first time the chain finalizes. This avoids + // some edge-cases around genesis. while rig.cached_head().finalized_checkpoint().epoch == 0 { rig.import_block(Payload::Syncing).await; } - let slots_per_epoch = E::slots_per_epoch(); - let start_slot = rig.cached_head().head_slot() + 1; - let mut opt_fork_block = None; - - assert_eq!(start_slot % slots_per_epoch, 1); - for i in 0..slots_per_epoch - 1 { - let slot = start_slot + i; - let slot_offset = slot.as_u64() % slots_per_epoch; - - rig.harness.set_current_slot(slot); - - if slot_offset == slots_per_epoch - 1 { - // Optimistic head block right before epoch boundary. - let is_valid = Payload::Syncing; - rig.import_block_parametric(is_valid, is_valid, Some(slot), |error| { - matches!( - error, - BlockError::ExecutionPayloadError( - ExecutionPayloadError::RejectedByExecutionEngine { .. } - ) - ) - }) - .await; - } else if 3 * slot_offset < 2 * slots_per_epoch { - // Valid block in previous epoch. - rig.import_block(Payload::Valid).await; - } else if slot_offset == slots_per_epoch - 2 { - // Fork block one slot prior to invalid head, not applied immediately. - let parent_state = rig - .harness - .chain - .state_at_slot(slot - 1, StateSkipConfig::WithStateRoots) - .unwrap(); - let (fork_block_tuple, _) = rig.harness.make_block(parent_state, slot).await; - opt_fork_block = Some(fork_block_tuple.0); - } else { - // Skipped slot. - }; + // Define a helper function. + let chain = rig.harness.chain.clone(); + let get_unrealized_justified_epoch = move || { + chain + .canonical_head + .fork_choice_read_lock() + .unrealized_justified_checkpoint() + .epoch + }; + + // Import more blocks until there is a new and higher unrealized + // justified checkpoint. + // + // The result will be a single chain where the head block has a higher + // unrealized justified checkpoint than all other blocks in the chain. + let initial_unrealized_justified = get_unrealized_justified_epoch(); + while get_unrealized_justified_epoch() == initial_unrealized_justified { + rig.import_block(Payload::Syncing).await; } + // Create a forked block that competes with the head block. Both the + // head block and this fork block will share the same parent. + // + // The fork block and head block will both have an unrealized justified + // checkpoint at epoch `N` whilst their parent is at `N - 1`. + let head_slot = rig.cached_head().head_slot(); + let parent_slot = head_slot - 1; + let fork_block_slot = head_slot + 1; + let parent_state = rig + .harness + .chain + .state_at_slot(parent_slot, StateSkipConfig::WithStateRoots) + .unwrap(); + let (fork_block_tuple, _) = rig.harness.make_block(parent_state, fork_block_slot).await; + let fork_block = fork_block_tuple.0; + let invalid_head = rig.cached_head(); - assert_eq!( - invalid_head.head_slot() % slots_per_epoch, - slots_per_epoch - 1 - ); - // Advance clock to new epoch to realize the justification of soon-to-be-invalid head block. - rig.harness.set_current_slot(invalid_head.head_slot() + 1); + // Advance the chain forward two epochs past the current head block. + // + // This ensures that `voting_source.epoch + 2 >= current_epoch` is + // `false` in the `node_is_viable_for_head` function. In effect, this + // ensures that no other block but the current head block is viable as a + // head block. + let invalid_head_epoch = invalid_head.head_slot().epoch(slots_per_epoch); + let new_wall_clock_epoch = invalid_head_epoch + 2; + rig.harness + .set_current_slot(new_wall_clock_epoch.start_slot(slots_per_epoch)); // Invalidate the head block. rig.invalidate_manually(invalid_head.head_block_root()) .await; + // Since our setup ensures that there is only a single, invalid block + // that's viable for head (according to FFG filtering), setting the + // head block as invalid should not result in another head being chosen. + // Rather, it should fail to run fork choice and leave the invalid block as + // the head. assert!(rig .canonical_head() .head_execution_status() .unwrap() .is_invalid()); - // Finding a new head should fail since the only possible head is not valid. + // Ensure that we're getting the correct error when trying to find a new + // head. rig.assert_get_head_error_contains("InvalidBestNode"); Self { rig, - fork_block: opt_fork_block.unwrap(), + fork_block, invalid_head, } } diff --git a/consensus/proto_array/src/fork_choice_test_definition/ffg_updates.rs b/consensus/proto_array/src/fork_choice_test_definition/ffg_updates.rs index 77211a86a7d..3b31616145d 100644 --- a/consensus/proto_array/src/fork_choice_test_definition/ffg_updates.rs +++ b/consensus/proto_array/src/fork_choice_test_definition/ffg_updates.rs @@ -59,7 +59,7 @@ pub fn get_ffg_case_01_test_definition() -> ForkChoiceTestDefinition { expected_head: get_root(3), }); - // Ensure that with justified epoch 1 we find 2 + // Ensure that with justified epoch 1 we find 3 // // 0 // | @@ -68,11 +68,15 @@ pub fn get_ffg_case_01_test_definition() -> ForkChoiceTestDefinition { // 2 <- start // | // 3 <- head + // + // Since https://github.com/ethereum/consensus-specs/pull/3431 it is valid + // to elect head blocks that have a higher justified checkpoint than the + // store. ops.push(Operation::FindHead { justified_checkpoint: get_checkpoint(1), finalized_checkpoint: get_checkpoint(0), justified_state_balances: balances.clone(), - expected_head: get_root(2), + expected_head: get_root(3), }); // Ensure that with justified epoch 2 we find 3 @@ -247,14 +251,19 @@ pub fn get_ffg_case_02_test_definition() -> ForkChoiceTestDefinition { justified_state_balances: balances.clone(), expected_head: get_root(10), }); - // Same as above, but with justified epoch 3 (should be invalid). - ops.push(Operation::InvalidFindHead { + // Same as above, but with justified epoch 3. + // + // Since https://github.com/ethereum/consensus-specs/pull/3431 it is valid + // to elect head blocks that have a higher justified checkpoint than the + // store. + ops.push(Operation::FindHead { justified_checkpoint: Checkpoint { epoch: Epoch::new(3), root: get_root(6), }, finalized_checkpoint: get_checkpoint(0), justified_state_balances: balances.clone(), + expected_head: get_root(10), }); // Add a vote to 1. @@ -305,14 +314,19 @@ pub fn get_ffg_case_02_test_definition() -> ForkChoiceTestDefinition { justified_state_balances: balances.clone(), expected_head: get_root(9), }); - // Save as above but justified epoch 3 (should fail). - ops.push(Operation::InvalidFindHead { + // Save as above but justified epoch 3. + // + // Since https://github.com/ethereum/consensus-specs/pull/3431 it is valid + // to elect head blocks that have a higher justified checkpoint than the + // store. + ops.push(Operation::FindHead { justified_checkpoint: Checkpoint { epoch: Epoch::new(3), root: get_root(5), }, finalized_checkpoint: get_checkpoint(0), justified_state_balances: balances.clone(), + expected_head: get_root(9), }); // Add a vote to 2. @@ -363,14 +377,19 @@ pub fn get_ffg_case_02_test_definition() -> ForkChoiceTestDefinition { justified_state_balances: balances.clone(), expected_head: get_root(10), }); - // Same as above but justified epoch 3 (should fail). - ops.push(Operation::InvalidFindHead { + // Same as above but justified epoch 3. + // + // Since https://github.com/ethereum/consensus-specs/pull/3431 it is valid + // to elect head blocks that have a higher justified checkpoint than the + // store. + ops.push(Operation::FindHead { justified_checkpoint: Checkpoint { epoch: Epoch::new(3), root: get_root(6), }, finalized_checkpoint: get_checkpoint(0), justified_state_balances: balances.clone(), + expected_head: get_root(10), }); // Ensure that if we start at 1 we find 9 (just: 0, fin: 0). @@ -405,14 +424,19 @@ pub fn get_ffg_case_02_test_definition() -> ForkChoiceTestDefinition { justified_state_balances: balances.clone(), expected_head: get_root(9), }); - // Same as above but justified epoch 3 (should fail). - ops.push(Operation::InvalidFindHead { + // Same as above but justified epoch 3. + // + // Since https://github.com/ethereum/consensus-specs/pull/3431 it is valid + // to elect head blocks that have a higher justified checkpoint than the + // store. + ops.push(Operation::FindHead { justified_checkpoint: Checkpoint { epoch: Epoch::new(3), root: get_root(5), }, finalized_checkpoint: get_checkpoint(0), justified_state_balances: balances.clone(), + expected_head: get_root(9), }); // Ensure that if we start at 2 we find 10 (just: 0, fin: 0). @@ -444,14 +468,19 @@ pub fn get_ffg_case_02_test_definition() -> ForkChoiceTestDefinition { justified_state_balances: balances.clone(), expected_head: get_root(10), }); - // Same as above but justified epoch 3 (should fail). - ops.push(Operation::InvalidFindHead { + // Same as above but justified epoch 3. + // + // Since https://github.com/ethereum/consensus-specs/pull/3431 it is valid + // to elect head blocks that have a higher justified checkpoint than the + // store. + ops.push(Operation::FindHead { justified_checkpoint: Checkpoint { epoch: Epoch::new(3), root: get_root(6), }, finalized_checkpoint: get_checkpoint(0), justified_state_balances: balances, + expected_head: get_root(10), }); // END OF TESTS diff --git a/consensus/proto_array/src/fork_choice_test_definition/no_votes.rs b/consensus/proto_array/src/fork_choice_test_definition/no_votes.rs index a60b3e6b368..27a7969e49b 100644 --- a/consensus/proto_array/src/fork_choice_test_definition/no_votes.rs +++ b/consensus/proto_array/src/fork_choice_test_definition/no_votes.rs @@ -184,7 +184,7 @@ pub fn get_no_votes_test_definition() -> ForkChoiceTestDefinition { root: Hash256::zero(), }, }, - // Ensure the head is still 4 whilst the justified epoch is 0. + // Ensure the head is now 5 whilst the justified epoch is 0. // // 0 // / \ @@ -203,9 +203,10 @@ pub fn get_no_votes_test_definition() -> ForkChoiceTestDefinition { root: Hash256::zero(), }, justified_state_balances: balances.clone(), - expected_head: get_root(4), + expected_head: get_root(5), }, - // Ensure there is an error when starting from a block that has the wrong justified epoch. + // Ensure there is no error when starting from a block that has the + // wrong justified epoch. // // 0 // / \ @@ -214,7 +215,11 @@ pub fn get_no_votes_test_definition() -> ForkChoiceTestDefinition { // 4 3 // | // 5 <- starting from 5 with justified epoch 0 should error. - Operation::InvalidFindHead { + // + // Since https://github.com/ethereum/consensus-specs/pull/3431 it is valid + // to elect head blocks that have a higher justified checkpoint than the + // store. + Operation::FindHead { justified_checkpoint: Checkpoint { epoch: Epoch::new(1), root: get_root(5), @@ -224,6 +229,7 @@ pub fn get_no_votes_test_definition() -> ForkChoiceTestDefinition { root: Hash256::zero(), }, justified_state_balances: balances.clone(), + expected_head: get_root(5), }, // Set the justified epoch to 2 and the start block to 5 and ensure 5 is the head. // diff --git a/consensus/proto_array/src/proto_array.rs b/consensus/proto_array/src/proto_array.rs index 4726715a162..7c2ecfe26a6 100644 --- a/consensus/proto_array/src/proto_array.rs +++ b/consensus/proto_array/src/proto_array.rs @@ -961,16 +961,9 @@ impl ProtoArray { node_justified_checkpoint }; - let mut correct_justified = self.justified_checkpoint.epoch == genesis_epoch - || voting_source.epoch == self.justified_checkpoint.epoch; - - if let Some(node_unrealized_justified_checkpoint) = node.unrealized_justified_checkpoint { - if !correct_justified && self.justified_checkpoint.epoch + 1 == current_epoch { - correct_justified = node_unrealized_justified_checkpoint.epoch - >= self.justified_checkpoint.epoch - && voting_source.epoch + 2 >= current_epoch; - } - } + let correct_justified = self.justified_checkpoint.epoch == genesis_epoch + || voting_source.epoch == self.justified_checkpoint.epoch + || voting_source.epoch + 2 >= current_epoch; let correct_finalized = self.finalized_checkpoint.epoch == genesis_epoch || self.is_finalized_checkpoint_or_descendant::(node.root); diff --git a/testing/ef_tests/Makefile b/testing/ef_tests/Makefile index e42db1801df..508c284275a 100644 --- a/testing/ef_tests/Makefile +++ b/testing/ef_tests/Makefile @@ -1,4 +1,4 @@ -TESTS_TAG := v1.4.0-beta.4 +TESTS_TAG := v1.4.0-beta.6 TESTS = general minimal mainnet TARBALLS = $(patsubst %,%-$(TESTS_TAG).tar.gz,$(TESTS)) diff --git a/testing/ef_tests/check_all_files_accessed.py b/testing/ef_tests/check_all_files_accessed.py index a5ab897c33f..1d1f2fa49a0 100755 --- a/testing/ef_tests/check_all_files_accessed.py +++ b/testing/ef_tests/check_all_files_accessed.py @@ -51,7 +51,8 @@ "bls12-381-tests/deserialization_G1", "bls12-381-tests/deserialization_G2", "bls12-381-tests/hash_to_G2", - "tests/.*/eip6110" + "tests/.*/eip6110", + "tests/.*/whisk" ] diff --git a/testing/ef_tests/tests/tests.rs b/testing/ef_tests/tests/tests.rs index dd25dba8b60..5ed657c6522 100644 --- a/testing/ef_tests/tests/tests.rs +++ b/testing/ef_tests/tests/tests.rs @@ -1,6 +1,6 @@ #![cfg(feature = "ef_tests")] -use ef_tests::{KzgInclusionMerkleProofValidityHandler, *}; +use ef_tests::*; use types::{MainnetEthSpec, MinimalEthSpec, *}; // Check that the hand-computed multiplications on EthSpec are correctly computed. @@ -605,6 +605,7 @@ fn merkle_proof_validity() { } #[test] +#[cfg(feature = "fake_crypto")] fn kzg_inclusion_merkle_proof_validity() { KzgInclusionMerkleProofValidityHandler::::default().run(); KzgInclusionMerkleProofValidityHandler::::default().run(); From e7ef2a3a54a871bac30ca1ce8a1382dd567eab8d Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Fri, 9 Feb 2024 06:59:39 +0200 Subject: [PATCH 10/24] validator liveness endpoint should accept string encoded indices (#5184) * deserialize string indices as u64 * client should send quoted indices --- beacon_node/http_api/src/lib.rs | 5 +++-- common/eth2/src/lib.rs | 10 +++++++--- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index ad841b20e3a..ec3fd494d49 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -3864,7 +3864,7 @@ pub fn serve( }, ); - // POST vaidator/liveness/{epoch} + // POST validator/liveness/{epoch} let post_validator_liveness_epoch = eth_v1 .and(warp::path("validator")) .and(warp::path("liveness")) @@ -3875,7 +3875,7 @@ pub fn serve( .and(chain_filter.clone()) .then( |epoch: Epoch, - indices: Vec, + indices: api_types::ValidatorIndexData, task_spawner: TaskSpawner, chain: Arc>| { task_spawner.blocking_json_task(Priority::P0, move || { @@ -3894,6 +3894,7 @@ pub fn serve( } let liveness: Vec = indices + .0 .iter() .cloned() .map(|index| { diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 6687788d528..3c22c822b8a 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -2258,7 +2258,7 @@ impl BeaconNodeHttpClient { pub async fn post_validator_liveness_epoch( &self, epoch: Epoch, - indices: &Vec, + indices: &[u64], ) -> Result>, Error> { let mut path = self.eth_path(V1)?; @@ -2268,8 +2268,12 @@ impl BeaconNodeHttpClient { .push("liveness") .push(&epoch.to_string()); - self.post_with_timeout_and_response(path, indices, self.timeouts.liveness) - .await + self.post_with_timeout_and_response( + path, + &ValidatorIndexDataRef(indices), + self.timeouts.liveness, + ) + .await } /// `POST validator/duties/attester/{epoch}` From 256d9042d35837740e05741b467afd6b23b476c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Thu, 15 Feb 2024 00:41:52 +0000 Subject: [PATCH 11/24] Drop gossipsub stale messages when polling ConnectionHandler. (#5175) * drop gossipsub stale messages * convert async-channel::Receiver to Peekable, to be able to Peek next message without dropping it --- .../src/gossipsub/behaviour/tests.rs | 843 ++++++++++-------- .../src/gossipsub/handler.rs | 32 +- .../lighthouse_network/src/gossipsub/types.rs | 110 ++- 3 files changed, 548 insertions(+), 437 deletions(-) diff --git a/beacon_node/lighthouse_network/src/gossipsub/behaviour/tests.rs b/beacon_node/lighthouse_network/src/gossipsub/behaviour/tests.rs index 9d8a10bcc85..eb006e52928 100644 --- a/beacon_node/lighthouse_network/src/gossipsub/behaviour/tests.rs +++ b/beacon_node/lighthouse_network/src/gossipsub/behaviour/tests.rs @@ -95,7 +95,7 @@ where // build and connect peer_no random peers let mut peers = vec![]; - let mut receiver_queues = HashMap::new(); + let mut receivers = HashMap::new(); let empty = vec![]; for i in 0..self.peer_no { @@ -110,10 +110,10 @@ where i < self.explicit, ); peers.push(peer); - receiver_queues.insert(peer, receiver); + receivers.insert(peer, receiver); } - (gs, peers, receiver_queues, topic_hashes) + (gs, peers, receivers, topic_hashes) } fn peer_no(mut self, peer_no: usize) -> Self { @@ -420,7 +420,7 @@ fn test_subscribe() { // - run JOIN(topic) let subscribe_topic = vec![String::from("test_subscribe")]; - let (gs, _, queues, topic_hashes) = inject_nodes1() + let (gs, _, receivers, topic_hashes) = inject_nodes1() .peer_no(20) .topics(subscribe_topic) .to_subscribe(true) @@ -432,11 +432,12 @@ fn test_subscribe() { ); // collect all the subscriptions - let subscriptions = queues + let subscriptions = receivers .into_values() .fold(0, |mut collected_subscriptions, c| { - while !c.priority.is_empty() { - if let Ok(RpcOut::Subscribe(_)) = c.priority.try_recv() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Subscribe(_)) = priority.try_recv() { collected_subscriptions += 1 } } @@ -447,8 +448,8 @@ fn test_subscribe() { assert_eq!(subscriptions, 20); } -#[test] /// Test unsubscribe. +#[test] fn test_unsubscribe() { // Unsubscribe should: // - Remove the mesh entry for topic @@ -462,7 +463,7 @@ fn test_unsubscribe() { .collect::>(); // subscribe to topic_strings - let (mut gs, _, queues, topic_hashes) = inject_nodes1() + let (mut gs, _, receivers, topic_hashes) = inject_nodes1() .peer_no(20) .topics(topic_strings) .to_subscribe(true) @@ -492,11 +493,12 @@ fn test_unsubscribe() { ); // collect all the subscriptions - let subscriptions = queues + let subscriptions = receivers .into_values() .fold(0, |mut collected_subscriptions, c| { - while !c.priority.is_empty() { - if let Ok(RpcOut::Subscribe(_)) = c.priority.try_recv() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Subscribe(_)) = priority.try_recv() { collected_subscriptions += 1 } } @@ -515,8 +517,8 @@ fn test_unsubscribe() { } } -#[test] /// Test JOIN(topic) functionality. +#[test] fn test_join() { // The Join function should: // - Remove peers from fanout[topic] @@ -540,7 +542,7 @@ fn test_join() { .create_network(); // Flush previous GRAFT messages. - flush_events(&mut gs, &receivers); + receivers = flush_events(&mut gs, receivers); // unsubscribe, then call join to invoke functionality assert!( @@ -564,17 +566,33 @@ fn test_join() { "Should have added 6 nodes to the mesh" ); - fn count_grafts(mut acc: usize, receiver: &RpcReceiver) -> usize { - while !receiver.priority.is_empty() || !receiver.non_priority.is_empty() { - if let Ok(RpcOut::Graft(_)) = receiver.priority.try_recv() { - acc += 1; + fn count_grafts( + receivers: HashMap, + ) -> (usize, HashMap) { + let mut new_receivers = HashMap::new(); + let mut acc = 0; + + for (peer_id, c) in receivers.into_iter() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Graft(_)) = priority.try_recv() { + acc += 1; + } } + new_receivers.insert( + peer_id, + RpcReceiver { + priority_len: c.priority_len, + priority: priority.peekable(), + non_priority: c.non_priority, + }, + ); } - acc + (acc, new_receivers) } // there should be mesh_n GRAFT messages. - let graft_messages = receivers.values().fold(0, count_grafts); + let (graft_messages, mut receivers) = count_grafts(receivers); assert_eq!( graft_messages, 6, @@ -645,12 +663,12 @@ fn test_join() { ); } - // there should now be 12 graft messages to be sent - let graft_messages = receivers.values().fold(graft_messages, count_grafts); + // there should now 6 graft messages to be sent + let (graft_messages, _) = count_grafts(receivers); assert_eq!( - graft_messages, 12, - "There should be 12 grafts messages sent to peers" + graft_messages, 6, + "There should be 6 grafts messages sent to peers" ); } @@ -668,7 +686,7 @@ fn test_publish_without_flood_publishing() { .unwrap(); let publish_topic = String::from("test_publish"); - let (mut gs, _, queues, topic_hashes) = inject_nodes1() + let (mut gs, _, receivers, topic_hashes) = inject_nodes1() .peer_no(20) .topics(vec![publish_topic.clone()]) .to_subscribe(true) @@ -695,11 +713,12 @@ fn test_publish_without_flood_publishing() { gs.publish(Topic::new(publish_topic), publish_data).unwrap(); // Collect all publish messages - let publishes = queues + let publishes = receivers .into_values() .fold(vec![], |mut collected_publish, c| { - while !c.priority.is_empty() { - if let Ok(RpcOut::Publish { message, .. }) = c.priority.try_recv() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Publish { message, .. }) = priority.try_recv() { collected_publish.push(message); } } @@ -747,7 +766,7 @@ fn test_fanout() { .unwrap(); let fanout_topic = String::from("test_fanout"); - let (mut gs, _, queues, topic_hashes) = inject_nodes1() + let (mut gs, _, receivers, topic_hashes) = inject_nodes1() .peer_no(20) .topics(vec![fanout_topic.clone()]) .to_subscribe(true) @@ -779,11 +798,12 @@ fn test_fanout() { ); // Collect all publish messages - let publishes = queues + let publishes = receivers .into_values() .fold(vec![], |mut collected_publish, c| { - while !c.priority.is_empty() { - if let Ok(RpcOut::Publish { message, .. }) = c.priority.try_recv() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Publish { message, .. }) = priority.try_recv() { collected_publish.push(message); } } @@ -815,10 +835,10 @@ fn test_fanout() { ); } -#[test] /// Test the gossipsub NetworkBehaviour peer connection logic. +#[test] fn test_inject_connected() { - let (gs, peers, queues, topic_hashes) = inject_nodes1() + let (gs, peers, receivers, topic_hashes) = inject_nodes1() .peer_no(20) .topics(vec![String::from("topic1"), String::from("topic2")]) .to_subscribe(true) @@ -826,11 +846,12 @@ fn test_inject_connected() { // check that our subscriptions are sent to each of the peers // collect all the SendEvents - let subscriptions = queues.into_iter().fold( + let subscriptions = receivers.into_iter().fold( HashMap::>::new(), |mut collected_subscriptions, (peer, c)| { - while !c.priority.is_empty() { - if let Ok(RpcOut::Subscribe(topic)) = c.priority.try_recv() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Subscribe(topic)) = priority.try_recv() { let mut peer_subs = collected_subscriptions.remove(&peer).unwrap_or_default(); peer_subs.push(topic.into_string()); collected_subscriptions.insert(peer, peer_subs); @@ -860,8 +881,8 @@ fn test_inject_connected() { } } -#[test] /// Test subscription handling +#[test] fn test_handle_received_subscriptions() { // For every subscription: // SUBSCRIBE: - Add subscribed topic to peer_topics for peer. @@ -972,8 +993,8 @@ fn test_handle_received_subscriptions() { ); } -#[test] /// Test Gossipsub.get_random_peers() function +#[test] fn test_get_random_peers() { // generate a default Config let gs_config = ConfigBuilder::default() @@ -1031,7 +1052,7 @@ fn test_get_random_peers() { /// Tests that the correct message is sent when a peer asks for a message in our cache. #[test] fn test_handle_iwant_msg_cached() { - let (mut gs, peers, queues, _) = inject_nodes1() + let (mut gs, peers, receivers, _) = inject_nodes1() .peer_no(20) .topics(Vec::new()) .to_subscribe(true) @@ -1059,11 +1080,12 @@ fn test_handle_iwant_msg_cached() { gs.handle_iwant(&peers[7], vec![msg_id.clone()]); // the messages we are sending - let sent_messages = queues + let sent_messages = receivers .into_values() .fold(vec![], |mut collected_messages, c| { - while !c.non_priority.is_empty() { - if let Ok(RpcOut::Forward { message, .. }) = c.non_priority.try_recv() { + let non_priority = c.non_priority.into_inner(); + while !non_priority.is_empty() { + if let Ok(RpcOut::Forward { message, .. }) = non_priority.try_recv() { collected_messages.push(message) } } @@ -1082,7 +1104,7 @@ fn test_handle_iwant_msg_cached() { /// Tests that messages are sent correctly depending on the shifting of the message cache. #[test] fn test_handle_iwant_msg_cached_shifted() { - let (mut gs, peers, queues, _) = inject_nodes1() + let (mut gs, peers, mut receivers, _) = inject_nodes1() .peer_no(20) .topics(Vec::new()) .to_subscribe(true) @@ -1115,21 +1137,29 @@ fn test_handle_iwant_msg_cached_shifted() { gs.handle_iwant(&peers[7], vec![msg_id.clone()]); // is the message is being sent? - let message_exists = queues.values().any(|c| { - let mut out = false; - while !c.non_priority.is_empty() { - if matches!(c.non_priority.try_recv(), Ok(RpcOut::Forward{message, timeout: _ }) if + let mut message_exists = false; + receivers = receivers.into_iter().map(|(peer_id, c)| { + let non_priority = c.non_priority.into_inner(); + while !non_priority.is_empty() { + if matches!(non_priority.try_recv(), Ok(RpcOut::Forward{message, timeout: _ }) if gs.config.message_id( &gs.data_transform .inbound_transform(message.clone()) .unwrap(), ) == msg_id) { - out = true; + message_exists = true; } } - out - }); + ( + peer_id, + RpcReceiver { + priority_len: c.priority_len, + priority: c.priority, + non_priority: non_priority.peekable(), + }, + ) + }).collect(); // default history_length is 5, expect no messages after shift > 5 if shift < 5 { assert!( @@ -1145,8 +1175,8 @@ fn test_handle_iwant_msg_cached_shifted() { } } +/// tests that an event is not created when a peers asks for a message not in our cache #[test] -// tests that an event is not created when a peers asks for a message not in our cache fn test_handle_iwant_msg_not_cached() { let (mut gs, peers, _, _) = inject_nodes1() .peer_no(20) @@ -1164,10 +1194,10 @@ fn test_handle_iwant_msg_not_cached() { ); } +/// tests that an event is created when a peer shares that it has a message we want #[test] -// tests that an event is created when a peer shares that it has a message we want fn test_handle_ihave_subscribed_and_msg_not_cached() { - let (mut gs, peers, receivers, topic_hashes) = inject_nodes1() + let (mut gs, peers, mut receivers, topic_hashes) = inject_nodes1() .peer_no(20) .topics(vec![String::from("topic1")]) .to_subscribe(true) @@ -1180,9 +1210,10 @@ fn test_handle_ihave_subscribed_and_msg_not_cached() { // check that we sent an IWANT request for `unknown id` let mut iwant_exists = false; - let receiver = receivers.get(&peers[7]).unwrap(); - while !receiver.non_priority.is_empty() { - if let Ok(RpcOut::IWant(IWant { message_ids })) = receiver.non_priority.try_recv() { + let receiver = receivers.remove(&peers[7]).unwrap(); + let non_priority = receiver.non_priority.into_inner(); + while !non_priority.is_empty() { + if let Ok(RpcOut::IWant(IWant { message_ids })) = non_priority.try_recv() { if message_ids .iter() .any(|m| *m == MessageId::new(b"unknown id")) @@ -1199,9 +1230,9 @@ fn test_handle_ihave_subscribed_and_msg_not_cached() { ); } +/// tests that an event is not created when a peer shares that it has a message that +/// we already have #[test] -// tests that an event is not created when a peer shares that it has a message that -// we already have fn test_handle_ihave_subscribed_and_msg_cached() { let (mut gs, peers, _, topic_hashes) = inject_nodes1() .peer_no(20) @@ -1221,9 +1252,9 @@ fn test_handle_ihave_subscribed_and_msg_cached() { ) } +/// test that an event is not created when a peer shares that it has a message in +/// a topic that we are not subscribed to #[test] -// test that an event is not created when a peer shares that it has a message in -// a topic that we are not subscribed to fn test_handle_ihave_not_subscribed() { let (mut gs, peers, _, _) = inject_nodes1() .peer_no(20) @@ -1247,9 +1278,9 @@ fn test_handle_ihave_not_subscribed() { ) } +/// tests that a peer is added to our mesh when we are both subscribed +/// to the same topic #[test] -// tests that a peer is added to our mesh when we are both subscribed -// to the same topic fn test_handle_graft_is_subscribed() { let (mut gs, peers, _, topic_hashes) = inject_nodes1() .peer_no(20) @@ -1265,9 +1296,9 @@ fn test_handle_graft_is_subscribed() { ); } +/// tests that a peer is not added to our mesh when they are subscribed to +/// a topic that we are not #[test] -// tests that a peer is not added to our mesh when they are subscribed to -// a topic that we are not fn test_handle_graft_is_not_subscribed() { let (mut gs, peers, _, topic_hashes) = inject_nodes1() .peer_no(20) @@ -1286,8 +1317,8 @@ fn test_handle_graft_is_not_subscribed() { ); } +/// tests multiple topics in a single graft message #[test] -// tests multiple topics in a single graft message fn test_handle_graft_multiple_topics() { let topics: Vec = ["topic1", "topic2", "topic3", "topic4"] .iter() @@ -1321,8 +1352,8 @@ fn test_handle_graft_multiple_topics() { ); } +/// tests that a peer is removed from our mesh #[test] -// tests that a peer is removed from our mesh fn test_handle_prune_peer_in_mesh() { let (mut gs, peers, _, topic_hashes) = inject_nodes1() .peer_no(20) @@ -1352,43 +1383,65 @@ fn test_handle_prune_peer_in_mesh() { } fn count_control_msgs( - queues: &HashMap, + receivers: HashMap, mut filter: impl FnMut(&PeerId, &RpcOut) -> bool, -) -> usize { - queues - .iter() - .fold(0, |mut collected_messages, (peer_id, c)| { - while !c.priority.is_empty() || !c.non_priority.is_empty() { - if let Ok(rpc) = c.priority.try_recv() { - if filter(peer_id, &rpc) { - collected_messages += 1; - } +) -> (usize, HashMap) { + let mut new_receivers = HashMap::new(); + let mut collected_messages = 0; + for (peer_id, c) in receivers.into_iter() { + let priority = c.priority.into_inner(); + let non_priority = c.non_priority.into_inner(); + while !priority.is_empty() || !non_priority.is_empty() { + if let Ok(rpc) = priority.try_recv() { + if filter(&peer_id, &rpc) { + collected_messages += 1; } - if let Ok(rpc) = c.non_priority.try_recv() { - if filter(peer_id, &rpc) { - collected_messages += 1; - } + } + if let Ok(rpc) = non_priority.try_recv() { + if filter(&peer_id, &rpc) { + collected_messages += 1; } } - collected_messages - }) + } + new_receivers.insert( + peer_id, + RpcReceiver { + priority_len: c.priority_len, + priority: priority.peekable(), + non_priority: non_priority.peekable(), + }, + ); + } + (collected_messages, new_receivers) } fn flush_events( gs: &mut Behaviour, - receiver_queues: &HashMap, -) { + receivers: HashMap, +) -> HashMap { gs.events.clear(); - for c in receiver_queues.values() { - while !c.priority.is_empty() || !c.non_priority.is_empty() { - let _ = c.priority.try_recv(); - let _ = c.non_priority.try_recv(); + let mut new_receivers = HashMap::new(); + for (peer_id, c) in receivers.into_iter() { + let priority = c.priority.into_inner(); + let non_priority = c.non_priority.into_inner(); + while !priority.is_empty() || !non_priority.is_empty() { + let _ = priority.try_recv(); + let _ = non_priority.try_recv(); } + new_receivers.insert( + peer_id, + RpcReceiver { + priority_len: c.priority_len, + priority: priority.peekable(), + non_priority: non_priority.peekable(), + }, + ); } + new_receivers } +/// tests that a peer added as explicit peer gets connected to #[test] -// tests that a peer added as explicit peer gets connected to fn test_explicit_peer_gets_connected() { let (mut gs, _, _, _) = inject_nodes1() .peer_no(0) @@ -1423,7 +1476,7 @@ fn test_explicit_peer_reconnects() { .check_explicit_peers_ticks(2) .build() .unwrap(); - let (mut gs, others, queues, _) = inject_nodes1() + let (mut gs, others, receivers, _) = inject_nodes1() .peer_no(1) .topics(Vec::new()) .to_subscribe(true) @@ -1435,7 +1488,7 @@ fn test_explicit_peer_reconnects() { //add peer as explicit peer gs.add_explicit_peer(peer); - flush_events(&mut gs, &queues); + flush_events(&mut gs, receivers); //disconnect peer disconnect_peer(&mut gs, peer); @@ -1473,7 +1526,7 @@ fn test_explicit_peer_reconnects() { #[test] fn test_handle_graft_explicit_peer() { - let (mut gs, peers, queues, topic_hashes) = inject_nodes1() + let (mut gs, peers, receivers, topic_hashes) = inject_nodes1() .peer_no(1) .topics(vec![String::from("topic1"), String::from("topic2")]) .to_subscribe(true) @@ -1490,21 +1543,24 @@ fn test_handle_graft_explicit_peer() { assert!(gs.mesh[&topic_hashes[1]].is_empty()); //check prunes - assert!( - count_control_msgs(&queues, |peer_id, m| peer_id == peer + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == peer && match m { - RpcOut::Prune(Prune { topic_hash, .. }) => - topic_hash == &topic_hashes[0] || topic_hash == &topic_hashes[1], + RpcOut::Prune(Prune { topic_hash, .. }) => { + topic_hash == &topic_hashes[0] || topic_hash == &topic_hashes[1] + } _ => false, - }) - >= 2, + } + }); + assert!( + control_msgs >= 2, "Not enough prunes sent when grafting from explicit peer" ); } #[test] fn explicit_peers_not_added_to_mesh_on_receiving_subscription() { - let (gs, peers, queues, topic_hashes) = inject_nodes1() + let (gs, peers, receivers, topic_hashes) = inject_nodes1() .peer_no(2) .topics(vec![String::from("topic1")]) .to_subscribe(true) @@ -1519,25 +1575,27 @@ fn explicit_peers_not_added_to_mesh_on_receiving_subscription() { ); //assert that graft gets created to non-explicit peer + let (control_msgs, receivers) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[1] && matches!(m, RpcOut::Graft { .. }) + }); assert!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[1] - && matches!(m, RpcOut::Graft { .. })) - >= 1, + control_msgs >= 1, "No graft message got created to non-explicit peer" ); //assert that no graft gets created to explicit peer + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[0] && matches!(m, RpcOut::Graft { .. }) + }); assert_eq!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[0] - && matches!(m, RpcOut::Graft { .. })), - 0, + control_msgs, 0, "A graft message got created to an explicit peer" ); } #[test] fn do_not_graft_explicit_peer() { - let (mut gs, others, queues, topic_hashes) = inject_nodes1() + let (mut gs, others, receivers, topic_hashes) = inject_nodes1() .peer_no(1) .topics(vec![String::from("topic")]) .to_subscribe(true) @@ -1551,17 +1609,18 @@ fn do_not_graft_explicit_peer() { assert_eq!(gs.mesh[&topic_hashes[0]], BTreeSet::new()); //assert that no graft gets created to explicit peer + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &others[0] && matches!(m, RpcOut::Graft { .. }) + }); assert_eq!( - count_control_msgs(&queues, |peer_id, m| peer_id == &others[0] - && matches!(m, RpcOut::Graft { .. })), - 0, + control_msgs, 0, "A graft message got created to an explicit peer" ); } #[test] fn do_forward_messages_to_explicit_peers() { - let (mut gs, peers, queues, topic_hashes) = inject_nodes1() + let (mut gs, peers, receivers, topic_hashes) = inject_nodes1() .peer_no(2) .topics(vec![String::from("topic1"), String::from("topic2")]) .to_subscribe(true) @@ -1582,9 +1641,10 @@ fn do_forward_messages_to_explicit_peers() { }; gs.handle_received_message(message.clone(), &local_id); assert_eq!( - queues.into_iter().fold(0, |mut fwds, (peer_id, c)| { - while !c.non_priority.is_empty() { - if matches!(c.non_priority.try_recv(), Ok(RpcOut::Forward{message: m, timeout: _}) if peer_id == peers[0] && m.data == message.data) { + receivers.into_iter().fold(0, |mut fwds, (peer_id, c)| { + let non_priority = c.non_priority.into_inner(); + while !non_priority.is_empty() { + if matches!(non_priority.try_recv(), Ok(RpcOut::Forward{message: m, timeout: _}) if peer_id == peers[0] && m.data == message.data) { fwds +=1; } } @@ -1597,7 +1657,7 @@ fn do_forward_messages_to_explicit_peers() { #[test] fn explicit_peers_not_added_to_mesh_on_subscribe() { - let (mut gs, peers, queues, _) = inject_nodes1() + let (mut gs, peers, receivers, _) = inject_nodes1() .peer_no(2) .topics(Vec::new()) .to_subscribe(true) @@ -1625,25 +1685,27 @@ fn explicit_peers_not_added_to_mesh_on_subscribe() { assert_eq!(gs.mesh[&topic_hash], vec![peers[1]].into_iter().collect()); //assert that graft gets created to non-explicit peer + let (control_msgs, receivers) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[1] && matches!(m, RpcOut::Graft { .. }) + }); assert!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[1] - && matches!(m, RpcOut::Graft { .. })) - > 0, + control_msgs > 0, "No graft message got created to non-explicit peer" ); //assert that no graft gets created to explicit peer + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[0] && matches!(m, RpcOut::Graft { .. }) + }); assert_eq!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[0] - && matches!(m, RpcOut::Graft { .. })), - 0, + control_msgs, 0, "A graft message got created to an explicit peer" ); } #[test] fn explicit_peers_not_added_to_mesh_from_fanout_on_subscribe() { - let (mut gs, peers, queues, _) = inject_nodes1() + let (mut gs, peers, receivers, _) = inject_nodes1() .peer_no(2) .topics(Vec::new()) .to_subscribe(true) @@ -1674,25 +1736,27 @@ fn explicit_peers_not_added_to_mesh_from_fanout_on_subscribe() { assert_eq!(gs.mesh[&topic_hash], vec![peers[1]].into_iter().collect()); //assert that graft gets created to non-explicit peer + let (control_msgs, receivers) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[1] && matches!(m, RpcOut::Graft { .. }) + }); assert!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[1] - && matches!(m, RpcOut::Graft { .. })) - >= 1, + control_msgs >= 1, "No graft message got created to non-explicit peer" ); //assert that no graft gets created to explicit peer + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[0] && matches!(m, RpcOut::Graft { .. }) + }); assert_eq!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[0] - && matches!(m, RpcOut::Graft { .. })), - 0, + control_msgs, 0, "A graft message got created to an explicit peer" ); } #[test] fn no_gossip_gets_sent_to_explicit_peers() { - let (mut gs, peers, receivers, topic_hashes) = inject_nodes1() + let (mut gs, peers, mut receivers, topic_hashes) = inject_nodes1() .peer_no(2) .topics(vec![String::from("topic1"), String::from("topic2")]) .to_subscribe(true) @@ -1721,23 +1785,24 @@ fn no_gossip_gets_sent_to_explicit_peers() { } //assert that no gossip gets sent to explicit peer - let receiver = receivers.get(&peers[0]).unwrap(); + let receiver = receivers.remove(&peers[0]).unwrap(); let mut gossips = 0; - while !receiver.non_priority.is_empty() { - if let Ok(RpcOut::IHave(_)) = receiver.non_priority.try_recv() { + let non_priority = receiver.non_priority.into_inner(); + while !non_priority.is_empty() { + if let Ok(RpcOut::IHave(_)) = non_priority.try_recv() { gossips += 1; } } assert_eq!(gossips, 0, "Gossip got emitted to explicit peer"); } -// Tests the mesh maintenance addition +/// Tests the mesh maintenance addition #[test] fn test_mesh_addition() { let config: Config = Config::default(); // Adds mesh_low peers and PRUNE 2 giving us a deficit. - let (mut gs, peers, _queues, topics) = inject_nodes1() + let (mut gs, peers, _receivers, topics) = inject_nodes1() .peer_no(config.mesh_n() + 1) .topics(vec!["test".into()]) .to_subscribe(true) @@ -1765,7 +1830,7 @@ fn test_mesh_addition() { assert_eq!(gs.mesh.get(&topics[0]).unwrap().len(), config.mesh_n()); } -// Tests the mesh maintenance subtraction +/// Tests the mesh maintenance subtraction #[test] fn test_mesh_subtraction() { let config = Config::default(); @@ -1853,7 +1918,7 @@ fn test_send_px_and_backoff_in_prune() { let config: Config = Config::default(); //build mesh with enough peers for px - let (mut gs, peers, queues, topics) = inject_nodes1() + let (mut gs, peers, receivers, topics) = inject_nodes1() .peer_no(config.prune_peers() + 1) .topics(vec!["test".into()]) .to_subscribe(true) @@ -1869,24 +1934,25 @@ fn test_send_px_and_backoff_in_prune() { ); //check prune message - assert_eq!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[0] + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[0] && match m { RpcOut::Prune(Prune { topic_hash, peers, backoff, - }) => + }) => { topic_hash == &topics[0] && peers.len() == config.prune_peers() && //all peers are different peers.iter().collect::>().len() == config.prune_peers() && - backoff.unwrap() == config.prune_backoff().as_secs(), + backoff.unwrap() == config.prune_backoff().as_secs() + } _ => false, - }), - 1 - ); + } + }); + assert_eq!(control_msgs, 1); } #[test] @@ -1894,7 +1960,7 @@ fn test_prune_backoffed_peer_on_graft() { let config: Config = Config::default(); //build mesh with enough peers for px - let (mut gs, peers, queues, topics) = inject_nodes1() + let (mut gs, peers, receivers, topics) = inject_nodes1() .peer_no(config.prune_peers() + 1) .topics(vec!["test".into()]) .to_subscribe(true) @@ -1911,28 +1977,29 @@ fn test_prune_backoffed_peer_on_graft() { ); //ignore all messages until now - flush_events(&mut gs, &queues); + let receivers = flush_events(&mut gs, receivers); //handle graft gs.handle_graft(&peers[0], vec![topics[0].clone()]); //check prune message - assert_eq!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[0] + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[0] && match m { RpcOut::Prune(Prune { topic_hash, peers, backoff, - }) => + }) => { topic_hash == &topics[0] && //no px in this case peers.is_empty() && - backoff.unwrap() == config.prune_backoff().as_secs(), + backoff.unwrap() == config.prune_backoff().as_secs() + } _ => false, - }), - 1 - ); + } + }); + assert_eq!(control_msgs, 1); } #[test] @@ -1943,7 +2010,7 @@ fn test_do_not_graft_within_backoff_period() { .build() .unwrap(); //only one peer => mesh too small and will try to regraft as early as possible - let (mut gs, peers, queues, topics) = inject_nodes1() + let (mut gs, peers, receivers, topics) = inject_nodes1() .peer_no(1) .topics(vec!["test".into()]) .to_subscribe(true) @@ -1954,7 +2021,7 @@ fn test_do_not_graft_within_backoff_period() { gs.handle_prune(&peers[0], vec![(topics[0].clone(), Vec::new(), Some(1))]); //forget all events until now - flush_events(&mut gs, &queues); + let receivers = flush_events(&mut gs, receivers); //call heartbeat gs.heartbeat(); @@ -1967,9 +2034,10 @@ fn test_do_not_graft_within_backoff_period() { //Check that no graft got created (we have backoff_slack = 1 therefore one more heartbeat // is needed). + let (control_msgs, receivers) = + count_control_msgs(receivers, |_, m| matches!(m, RpcOut::Graft { .. })); assert_eq!( - count_control_msgs(&queues, |_, m| matches!(m, RpcOut::Graft { .. })), - 0, + control_msgs, 0, "Graft message created too early within backoff period" ); @@ -1978,8 +2046,9 @@ fn test_do_not_graft_within_backoff_period() { gs.heartbeat(); //check that graft got created + let (control_msgs, _) = count_control_msgs(receivers, |_, m| matches!(m, RpcOut::Graft { .. })); assert!( - count_control_msgs(&queues, |_, m| matches!(m, RpcOut::Graft { .. })) > 0, + control_msgs > 0, "No graft message was created after backoff period" ); } @@ -1994,7 +2063,7 @@ fn test_do_not_graft_within_default_backoff_period_after_receiving_prune_without .build() .unwrap(); //only one peer => mesh too small and will try to regraft as early as possible - let (mut gs, peers, queues, topics) = inject_nodes1() + let (mut gs, peers, receivers, topics) = inject_nodes1() .peer_no(1) .topics(vec!["test".into()]) .to_subscribe(true) @@ -2005,7 +2074,7 @@ fn test_do_not_graft_within_default_backoff_period_after_receiving_prune_without gs.handle_prune(&peers[0], vec![(topics[0].clone(), Vec::new(), None)]); //forget all events until now - flush_events(&mut gs, &queues); + let receivers = flush_events(&mut gs, receivers); //call heartbeat gs.heartbeat(); @@ -2016,9 +2085,10 @@ fn test_do_not_graft_within_default_backoff_period_after_receiving_prune_without //Check that no graft got created (we have backoff_slack = 1 therefore one more heartbeat // is needed). + let (control_msgs, receivers) = + count_control_msgs(receivers, |_, m| matches!(m, RpcOut::Graft { .. })); assert_eq!( - count_control_msgs(&queues, |_, m| matches!(m, RpcOut::Graft { .. })), - 0, + control_msgs, 0, "Graft message created too early within backoff period" ); @@ -2027,8 +2097,9 @@ fn test_do_not_graft_within_default_backoff_period_after_receiving_prune_without gs.heartbeat(); //check that graft got created + let (control_msgs, _) = count_control_msgs(receivers, |_, m| matches!(m, RpcOut::Graft { .. })); assert!( - count_control_msgs(&queues, |_, m| matches!(m, RpcOut::Graft { .. })) > 0, + control_msgs > 0, "No graft message was created after backoff period" ); } @@ -2047,7 +2118,7 @@ fn test_unsubscribe_backoff() { let topic = String::from("test"); // only one peer => mesh too small and will try to regraft as early as possible - let (mut gs, _, queues, topics) = inject_nodes1() + let (mut gs, _, receivers, topics) = inject_nodes1() .peer_no(1) .topics(vec![topic.clone()]) .to_subscribe(true) @@ -2056,19 +2127,19 @@ fn test_unsubscribe_backoff() { let _ = gs.unsubscribe(&Topic::new(topic)); + let (control_msgs, receivers) = count_control_msgs(receivers, |_, m| match m { + RpcOut::Prune(Prune { backoff, .. }) => backoff == &Some(1), + _ => false, + }); assert_eq!( - count_control_msgs(&queues, |_, m| match m { - RpcOut::Prune(Prune { backoff, .. }) => backoff == &Some(1), - _ => false, - }), - 1, + control_msgs, 1, "Peer should be pruned with `unsubscribe_backoff`." ); let _ = gs.subscribe(&Topic::new(topics[0].to_string())); // forget all events until now - flush_events(&mut gs, &queues); + let receivers = flush_events(&mut gs, receivers); // call heartbeat gs.heartbeat(); @@ -2081,9 +2152,10 @@ fn test_unsubscribe_backoff() { // Check that no graft got created (we have backoff_slack = 1 therefore one more heartbeat // is needed). + let (control_msgs, receivers) = + count_control_msgs(receivers, |_, m| matches!(m, RpcOut::Graft { .. })); assert_eq!( - count_control_msgs(&queues, |_, m| matches!(m, RpcOut::Graft { .. })), - 0, + control_msgs, 0, "Graft message created too early within backoff period" ); @@ -2092,8 +2164,9 @@ fn test_unsubscribe_backoff() { gs.heartbeat(); // check that graft got created + let (control_msgs, _) = count_control_msgs(receivers, |_, m| matches!(m, RpcOut::Graft { .. })); assert!( - count_control_msgs(&queues, |_, m| matches!(m, RpcOut::Graft { .. })) > 0, + control_msgs > 0, "No graft message was created after backoff period" ); } @@ -2104,7 +2177,7 @@ fn test_flood_publish() { let topic = "test"; // Adds more peers than mesh can hold to test flood publishing - let (mut gs, _, queues, _) = inject_nodes1() + let (mut gs, _, receivers, _) = inject_nodes1() .peer_no(config.mesh_n_high() + 10) .topics(vec![topic.into()]) .to_subscribe(true) @@ -2115,11 +2188,12 @@ fn test_flood_publish() { gs.publish(Topic::new(topic), publish_data).unwrap(); // Collect all publish messages - let publishes = queues + let publishes = receivers .into_values() .fold(vec![], |mut collected_publish, c| { - while !c.priority.is_empty() { - if let Ok(RpcOut::Publish { message, .. }) = c.priority.try_recv() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Publish { message, .. }) = priority.try_recv() { collected_publish.push(message); } } @@ -2158,7 +2232,7 @@ fn test_gossip_to_at_least_gossip_lazy_peers() { //add more peers than in mesh to test gossipping //by default only mesh_n_low peers will get added to mesh - let (mut gs, _, queues, topic_hashes) = inject_nodes1() + let (mut gs, _, receivers, topic_hashes) = inject_nodes1() .peer_no(config.mesh_n_low() + config.gossip_lazy() + 1) .topics(vec!["topic".into()]) .to_subscribe(true) @@ -2185,16 +2259,14 @@ fn test_gossip_to_at_least_gossip_lazy_peers() { let msg_id = gs.config.message_id(message); //check that exactly config.gossip_lazy() many gossip messages were sent. - assert_eq!( - count_control_msgs(&queues, |_, action| match action { - RpcOut::IHave(IHave { - topic_hash, - message_ids, - }) => topic_hash == &topic_hashes[0] && message_ids.iter().any(|id| id == &msg_id), - _ => false, - }), - config.gossip_lazy() - ); + let (control_msgs, _) = count_control_msgs(receivers, |_, action| match action { + RpcOut::IHave(IHave { + topic_hash, + message_ids, + }) => topic_hash == &topic_hashes[0] && message_ids.iter().any(|id| id == &msg_id), + _ => false, + }); + assert_eq!(control_msgs, config.gossip_lazy()); } #[test] @@ -2203,7 +2275,7 @@ fn test_gossip_to_at_most_gossip_factor_peers() { //add a lot of peers let m = config.mesh_n_low() + config.gossip_lazy() * (2.0 / config.gossip_factor()) as usize; - let (mut gs, _, queues, topic_hashes) = inject_nodes1() + let (mut gs, _, receivers, topic_hashes) = inject_nodes1() .peer_no(m) .topics(vec!["topic".into()]) .to_subscribe(true) @@ -2229,14 +2301,15 @@ fn test_gossip_to_at_most_gossip_factor_peers() { let msg_id = gs.config.message_id(message); //check that exactly config.gossip_lazy() many gossip messages were sent. + let (control_msgs, _) = count_control_msgs(receivers, |_, action| match action { + RpcOut::IHave(IHave { + topic_hash, + message_ids, + }) => topic_hash == &topic_hashes[0] && message_ids.iter().any(|id| id == &msg_id), + _ => false, + }); assert_eq!( - count_control_msgs(&queues, |_, action| match action { - RpcOut::IHave(IHave { - topic_hash, - message_ids, - }) => topic_hash == &topic_hashes[0] && message_ids.iter().any(|id| id == &msg_id), - _ => false, - }), + control_msgs, ((m - config.mesh_n_low()) as f64 * config.gossip_factor()) as usize ); } @@ -2365,7 +2438,7 @@ fn test_prune_negative_scored_peers() { let config = Config::default(); //build mesh with one peer - let (mut gs, peers, queues, topics) = inject_nodes1() + let (mut gs, peers, receivers, topics) = inject_nodes1() .peer_no(1) .topics(vec!["test".into()]) .to_subscribe(true) @@ -2388,22 +2461,23 @@ fn test_prune_negative_scored_peers() { assert!(gs.mesh[&topics[0]].is_empty()); //check prune message - assert_eq!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[0] + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[0] && match m { RpcOut::Prune(Prune { topic_hash, peers, backoff, - }) => + }) => { topic_hash == &topics[0] && //no px in this case peers.is_empty() && - backoff.unwrap() == config.prune_backoff().as_secs(), + backoff.unwrap() == config.prune_backoff().as_secs() + } _ => false, - }), - 1 - ); + } + }); + assert_eq!(control_msgs, 1); } #[test] @@ -2496,7 +2570,7 @@ fn test_only_send_nonnegative_scoring_peers_in_px() { .unwrap(); // Build mesh with three peer - let (mut gs, peers, queues, topics) = inject_nodes1() + let (mut gs, peers, receivers, topics) = inject_nodes1() .peer_no(3) .topics(vec!["test".into()]) .to_subscribe(true) @@ -2522,29 +2596,26 @@ fn test_only_send_nonnegative_scoring_peers_in_px() { ); // Check that px in prune message only contains third peer - assert_eq!( - count_control_msgs(&queues, |peer_id, m| peer_id == &peers[1] + let (control_msgs, _) = count_control_msgs(receivers, |peer_id, m| { + peer_id == &peers[1] && match m { RpcOut::Prune(Prune { topic_hash, peers: px, .. - }) => + }) => { topic_hash == &topics[0] && px.len() == 1 - && px[0].peer_id.as_ref().unwrap() == &peers[2], + && px[0].peer_id.as_ref().unwrap() == &peers[2] + } _ => false, - }), - 1 - ); + } + }); + assert_eq!(control_msgs, 1); } #[test] fn test_do_not_gossip_to_peers_below_gossip_threshold() { - // use tracing_subscriber::EnvFilter; - // let _ = tracing_subscriber::fmt() - // .with_env_filter(EnvFilter::from_default_env()) - // .try_init(); let config = Config::default(); let peer_score_params = PeerScoreParams::default(); let peer_score_thresholds = PeerScoreThresholds { @@ -2601,23 +2672,21 @@ fn test_do_not_gossip_to_peers_below_gossip_threshold() { gs.emit_gossip(); // Check that exactly one gossip messages got sent and it got sent to p2 - assert_eq!( - count_control_msgs(&receivers, |peer, action| match action { - RpcOut::IHave(IHave { - topic_hash, - message_ids, - }) => { - if topic_hash == &topics[0] && message_ids.iter().any(|id| id == &msg_id) { - assert_eq!(peer, &p2); - true - } else { - false - } + let (control_msgs, _) = count_control_msgs(receivers, |peer, action| match action { + RpcOut::IHave(IHave { + topic_hash, + message_ids, + }) => { + if topic_hash == &topics[0] && message_ids.iter().any(|id| id == &msg_id) { + assert_eq!(peer, &p2); + true + } else { + false } - _ => false, - }), - 1 - ); + } + _ => false, + }); + assert_eq!(control_msgs, 1); } #[test] @@ -2630,7 +2699,7 @@ fn test_iwant_msg_from_peer_below_gossip_threshold_gets_ignored() { }; // Build full mesh - let (mut gs, peers, mut queues, topics) = inject_nodes1() + let (mut gs, peers, mut receivers, topics) = inject_nodes1() .peer_no(config.mesh_n_high()) .topics(vec!["test".into()]) .to_subscribe(true) @@ -2647,9 +2716,9 @@ fn test_iwant_msg_from_peer_below_gossip_threshold_gets_ignored() { // Add two additional peers that will not be part of the mesh let (p1, receiver1) = add_peer(&mut gs, &topics, false, false); - queues.insert(p1, receiver1); + receivers.insert(p1, receiver1); let (p2, receiver2) = add_peer(&mut gs, &topics, false, false); - queues.insert(p2, receiver2); + receivers.insert(p2, receiver2); // Reduce score of p1 below peer_score_thresholds.gossip_threshold // note that penalties get squared so two penalties means a score of @@ -2680,16 +2749,18 @@ fn test_iwant_msg_from_peer_below_gossip_threshold_gets_ignored() { gs.handle_iwant(&p2, vec![msg_id.clone()]); // the messages we are sending - let sent_messages = queues - .into_iter() - .fold(vec![], |mut collected_messages, (peer_id, c)| { - while !c.non_priority.is_empty() { - if let Ok(RpcOut::Forward { message, .. }) = c.non_priority.try_recv() { - collected_messages.push((peer_id, message)); + let sent_messages = + receivers + .into_iter() + .fold(vec![], |mut collected_messages, (peer_id, c)| { + let non_priority = c.non_priority.into_inner(); + while !non_priority.is_empty() { + if let Ok(RpcOut::Forward { message, .. }) = non_priority.try_recv() { + collected_messages.push((peer_id, message)); + } } - } - collected_messages - }); + collected_messages + }); //the message got sent to p2 assert!(sent_messages @@ -2718,7 +2789,7 @@ fn test_ihave_msg_from_peer_below_gossip_threshold_gets_ignored() { ..PeerScoreThresholds::default() }; //build full mesh - let (mut gs, peers, mut queues, topics) = inject_nodes1() + let (mut gs, peers, mut receivers, topics) = inject_nodes1() .peer_no(config.mesh_n_high()) .topics(vec!["test".into()]) .to_subscribe(true) @@ -2735,9 +2806,9 @@ fn test_ihave_msg_from_peer_below_gossip_threshold_gets_ignored() { //add two additional peers that will not be part of the mesh let (p1, receiver1) = add_peer(&mut gs, &topics, false, false); - queues.insert(p1, receiver1); + receivers.insert(p1, receiver1); let (p2, receiver2) = add_peer(&mut gs, &topics, false, false); - queues.insert(p2, receiver2); + receivers.insert(p2, receiver2); //reduce score of p1 below peer_score_thresholds.gossip_threshold //note that penalties get squared so two penalties means a score of @@ -2767,19 +2838,18 @@ fn test_ihave_msg_from_peer_below_gossip_threshold_gets_ignored() { gs.handle_ihave(&p2, vec![(topics[0].clone(), vec![msg_id.clone()])]); // check that we sent exactly one IWANT request to p2 - assert_eq!( - count_control_msgs(&queues, |peer, c| match c { - RpcOut::IWant(IWant { message_ids }) => - if message_ids.iter().any(|m| m == &msg_id) { - assert_eq!(peer, &p2); - true - } else { - false - }, - _ => false, - }), - 1 - ); + let (control_msgs, _) = count_control_msgs(receivers, |peer, c| match c { + RpcOut::IWant(IWant { message_ids }) => { + if message_ids.iter().any(|m| m == &msg_id) { + assert_eq!(peer, &p2); + true + } else { + false + } + } + _ => false, + }); + assert_eq!(control_msgs, 1); } #[test] @@ -2796,7 +2866,7 @@ fn test_do_not_publish_to_peer_below_publish_threshold() { }; //build mesh with no peers and no subscribed topics - let (mut gs, _, mut queues, _) = inject_nodes1() + let (mut gs, _, mut receivers, _) = inject_nodes1() .gs_config(config) .scoring(Some((peer_score_params, peer_score_thresholds))) .create_network(); @@ -2807,9 +2877,9 @@ fn test_do_not_publish_to_peer_below_publish_threshold() { //add two additional peers that will be added to the mesh let (p1, receiver1) = add_peer(&mut gs, &topics, false, false); - queues.insert(p1, receiver1); + receivers.insert(p1, receiver1); let (p2, receiver2) = add_peer(&mut gs, &topics, false, false); - queues.insert(p2, receiver2); + receivers.insert(p2, receiver2); //reduce score of p1 below peer_score_thresholds.publish_threshold //note that penalties get squared so two penalties means a score of @@ -2827,11 +2897,12 @@ fn test_do_not_publish_to_peer_below_publish_threshold() { gs.publish(topic, publish_data).unwrap(); // Collect all publish messages - let publishes = queues + let publishes = receivers .into_iter() .fold(vec![], |mut collected_publish, (peer_id, c)| { - while !c.priority.is_empty() { - if let Ok(RpcOut::Publish { message, .. }) = c.priority.try_recv() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Publish { message, .. }) = priority.try_recv() { collected_publish.push((peer_id, message)); } } @@ -2853,7 +2924,7 @@ fn test_do_not_flood_publish_to_peer_below_publish_threshold() { ..PeerScoreThresholds::default() }; //build mesh with no peers - let (mut gs, _, mut queues, topics) = inject_nodes1() + let (mut gs, _, mut receivers, topics) = inject_nodes1() .topics(vec!["test".into()]) .gs_config(config) .scoring(Some((peer_score_params, peer_score_thresholds))) @@ -2861,9 +2932,9 @@ fn test_do_not_flood_publish_to_peer_below_publish_threshold() { //add two additional peers that will be added to the mesh let (p1, receiver1) = add_peer(&mut gs, &topics, false, false); - queues.insert(p1, receiver1); + receivers.insert(p1, receiver1); let (p2, receiver2) = add_peer(&mut gs, &topics, false, false); - queues.insert(p2, receiver2); + receivers.insert(p2, receiver2); //reduce score of p1 below peer_score_thresholds.publish_threshold //note that penalties get squared so two penalties means a score of @@ -2881,11 +2952,12 @@ fn test_do_not_flood_publish_to_peer_below_publish_threshold() { gs.publish(Topic::new("test"), publish_data).unwrap(); // Collect all publish messages - let publishes = queues + let publishes = receivers .into_iter() .fold(vec![], |mut collected_publish, (peer_id, c)| { - while !c.priority.is_empty() { - if let Ok(RpcOut::Publish { message, .. }) = c.priority.try_recv() { + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if let Ok(RpcOut::Publish { message, .. }) = priority.try_recv() { collected_publish.push((peer_id, message)) } } @@ -4351,7 +4423,7 @@ fn test_opportunistic_grafting() { #[test] fn test_ignore_graft_from_unknown_topic() { //build gossipsub without subscribing to any topics - let (mut gs, peers, queues, _) = inject_nodes1() + let (mut gs, peers, receivers, _) = inject_nodes1() .peer_no(1) .topics(vec![]) .to_subscribe(false) @@ -4361,9 +4433,9 @@ fn test_ignore_graft_from_unknown_topic() { gs.handle_graft(&peers[0], vec![Topic::new("test").hash()]); //assert that no prune got created + let (control_msgs, _) = count_control_msgs(receivers, |_, a| matches!(a, RpcOut::Prune { .. })); assert_eq!( - count_control_msgs(&queues, |_, a| matches!(a, RpcOut::Prune { .. })), - 0, + control_msgs, 0, "we should not prune after graft in unknown topic" ); } @@ -4372,7 +4444,7 @@ fn test_ignore_graft_from_unknown_topic() { fn test_ignore_too_many_iwants_from_same_peer_for_same_message() { let config = Config::default(); //build gossipsub with full mesh - let (mut gs, _, mut queues, topics) = inject_nodes1() + let (mut gs, _, mut receivers, topics) = inject_nodes1() .peer_no(config.mesh_n_high()) .topics(vec!["test".into()]) .to_subscribe(false) @@ -4380,7 +4452,7 @@ fn test_ignore_too_many_iwants_from_same_peer_for_same_message() { //add another peer not in the mesh let (peer, receiver) = add_peer(&mut gs, &topics, false, false); - queues.insert(peer, receiver); + receivers.insert(peer, receiver); //receive a message let mut seq = 0; @@ -4394,7 +4466,7 @@ fn test_ignore_too_many_iwants_from_same_peer_for_same_message() { gs.handle_received_message(m1, &PeerId::random()); //clear events - flush_events(&mut gs, &queues); + let receivers = flush_events(&mut gs, receivers); //the first gossip_retransimission many iwants return the valid message, all others are // ignored. @@ -4403,9 +4475,10 @@ fn test_ignore_too_many_iwants_from_same_peer_for_same_message() { } assert_eq!( - queues.into_values().fold(0, |mut fwds, c| { - while !c.non_priority.is_empty() { - if let Ok(RpcOut::Forward { .. }) = c.non_priority.try_recv() { + receivers.into_values().fold(0, |mut fwds, c| { + let non_priority = c.non_priority.into_inner(); + while !non_priority.is_empty() { + if let Ok(RpcOut::Forward { .. }) = non_priority.try_recv() { fwds += 1; } } @@ -4460,10 +4533,12 @@ fn test_ignore_too_many_ihaves() { .collect(); //we send iwant only for the first 10 messages + let (control_msgs, receivers) = count_control_msgs(receivers, |p, action| { + p == &peer + && matches!(action, RpcOut::IWant(IWant { message_ids }) if message_ids.len() == 1 && first_ten.contains(&message_ids[0])) + }); assert_eq!( - count_control_msgs(&receivers, |p, action| p == &peer - && matches!(action, RpcOut::IWant(IWant { message_ids }) if message_ids.len() == 1 && first_ten.contains(&message_ids[0]))), - 10, + control_msgs, 10, "exactly the first ten ihaves should be processed and one iwant for each created" ); @@ -4484,12 +4559,11 @@ fn test_ignore_too_many_ihaves() { } //we sent iwant for all 10 messages - assert_eq!( - count_control_msgs(&receivers, |p, action| p == &peer - && matches!(action, RpcOut::IWant(IWant { message_ids }) if message_ids.len() == 1)), - 10, - "all 20 should get sent" - ); + let (control_msgs, _) = count_control_msgs(receivers, |p, action| { + p == &peer + && matches!(action, RpcOut::IWant(IWant { message_ids }) if message_ids.len() == 1) + }); + assert_eq!(control_msgs, 10, "all 20 should get sent"); } #[test] @@ -4500,7 +4574,7 @@ fn test_ignore_too_many_messages_in_ihave() { .build() .unwrap(); //build gossipsub with full mesh - let (mut gs, _, mut queues, topics) = inject_nodes1() + let (mut gs, _, mut receivers, topics) = inject_nodes1() .peer_no(config.mesh_n_high()) .topics(vec!["test".into()]) .to_subscribe(false) @@ -4509,7 +4583,7 @@ fn test_ignore_too_many_messages_in_ihave() { //add another peer not in the mesh let (peer, receiver) = add_peer(&mut gs, &topics, false, false); - queues.insert(peer, receiver); + receivers.insert(peer, receiver); //peer has 20 messages let mut seq = 0; @@ -4534,18 +4608,18 @@ fn test_ignore_too_many_messages_in_ihave() { //we send iwant only for the first 10 messages let mut sum = 0; - assert_eq!( - count_control_msgs(&queues, |p, rpc| match rpc { - RpcOut::IWant(IWant { message_ids }) => { - p == &peer && { - assert!(first_twelve.is_superset(&message_ids.iter().collect())); - sum += message_ids.len(); - true - } + let (control_msgs, receivers) = count_control_msgs(receivers, |p, rpc| match rpc { + RpcOut::IWant(IWant { message_ids }) => { + p == &peer && { + assert!(first_twelve.is_superset(&message_ids.iter().collect())); + sum += message_ids.len(); + true } - _ => false, - }), - 2, + } + _ => false, + }); + assert_eq!( + control_msgs, 2, "the third ihave should get ignored and no iwant sent" ); @@ -4560,20 +4634,16 @@ fn test_ignore_too_many_messages_in_ihave() { //we sent 10 iwant messages ids via a IWANT rpc. let mut sum = 0; - assert_eq!( - count_control_msgs(&queues, |p, rpc| { - match rpc { - RpcOut::IWant(IWant { message_ids }) => { - p == &peer && { - sum += message_ids.len(); - true - } - } - _ => false, + let (control_msgs, _) = count_control_msgs(receivers, |p, rpc| match rpc { + RpcOut::IWant(IWant { message_ids }) => { + p == &peer && { + sum += message_ids.len(); + true } - }), - 1 - ); + } + _ => false, + }); + assert_eq!(control_msgs, 1); assert_eq!(sum, 10, "exactly 20 iwants should get sent"); } @@ -4619,22 +4689,22 @@ fn test_limit_number_of_message_ids_inside_ihave() { let mut ihaves1 = HashSet::new(); let mut ihaves2 = HashSet::new(); - assert_eq!( - count_control_msgs(&receivers, |p, action| match action { - RpcOut::IHave(IHave { message_ids, .. }) => { - if p == &p1 { - ihaves1 = message_ids.iter().cloned().collect(); - true - } else if p == &p2 { - ihaves2 = message_ids.iter().cloned().collect(); - true - } else { - false - } + let (control_msgs, _) = count_control_msgs(receivers, |p, action| match action { + RpcOut::IHave(IHave { message_ids, .. }) => { + if p == &p1 { + ihaves1 = message_ids.iter().cloned().collect(); + true + } else if p == &p2 { + ihaves2 = message_ids.iter().cloned().collect(); + true + } else { + false } - _ => false, - }), - 2, + } + _ => false, + }); + assert_eq!( + control_msgs, 2, "should have emitted one ihave to p1 and one to p2" ); @@ -4668,7 +4738,6 @@ fn test_iwant_penalties() { .with_env_filter(EnvFilter::from_default_env()) .try_init(); */ - let config = ConfigBuilder::default() .iwant_followup_time(Duration::from_secs(4)) .build() @@ -4791,7 +4860,7 @@ fn test_publish_to_floodsub_peers_without_flood_publish() { .flood_publish(false) .build() .unwrap(); - let (mut gs, _, mut queues, topics) = inject_nodes1() + let (mut gs, _, mut receivers, topics) = inject_nodes1() .peer_no(config.mesh_n_low() - 1) .topics(vec!["test".into()]) .to_subscribe(false) @@ -4807,11 +4876,11 @@ fn test_publish_to_floodsub_peers_without_flood_publish() { Multiaddr::empty(), Some(PeerKind::Floodsub), ); - queues.insert(p1, receiver1); + receivers.insert(p1, receiver1); let (p2, receiver2) = add_peer_with_addr_and_kind(&mut gs, &topics, false, false, Multiaddr::empty(), None); - queues.insert(p2, receiver2); + receivers.insert(p2, receiver2); //p1 and p2 are not in the mesh assert!(!gs.mesh[&topics[0]].contains(&p1) && !gs.mesh[&topics[0]].contains(&p2)); @@ -4821,12 +4890,13 @@ fn test_publish_to_floodsub_peers_without_flood_publish() { gs.publish(Topic::new("test"), publish_data).unwrap(); // Collect publish messages to floodsub peers - let publishes = queues - .iter() + let publishes = receivers + .into_iter() .fold(0, |mut collected_publish, (peer_id, c)| { - while !c.priority.is_empty() { - if matches!(c.priority.try_recv(), - Ok(RpcOut::Publish{..}) if peer_id == &p1 || peer_id == &p2) + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if matches!(priority.try_recv(), + Ok(RpcOut::Publish{..}) if peer_id == p1 || peer_id == p2) { collected_publish += 1; } @@ -4846,7 +4916,7 @@ fn test_do_not_use_floodsub_in_fanout() { .flood_publish(false) .build() .unwrap(); - let (mut gs, _, mut queues, _) = inject_nodes1() + let (mut gs, _, mut receivers, _) = inject_nodes1() .peer_no(config.mesh_n_low() - 1) .topics(Vec::new()) .to_subscribe(false) @@ -4866,22 +4936,23 @@ fn test_do_not_use_floodsub_in_fanout() { Some(PeerKind::Floodsub), ); - queues.insert(p1, receiver1); + receivers.insert(p1, receiver1); let (p2, receiver2) = add_peer_with_addr_and_kind(&mut gs, &topics, false, false, Multiaddr::empty(), None); - queues.insert(p2, receiver2); + receivers.insert(p2, receiver2); //publish a message let publish_data = vec![0; 42]; gs.publish(Topic::new("test"), publish_data).unwrap(); // Collect publish messages to floodsub peers - let publishes = queues - .iter() + let publishes = receivers + .into_iter() .fold(0, |mut collected_publish, (peer_id, c)| { - while !c.priority.is_empty() { - if matches!(c.priority.try_recv(), - Ok(RpcOut::Publish{..}) if peer_id == &p1 || peer_id == &p2) + let priority = c.priority.into_inner(); + while !priority.is_empty() { + if matches!(priority.try_recv(), + Ok(RpcOut::Publish{..}) if peer_id == p1 || peer_id == p2) { collected_publish += 1; } @@ -4932,7 +5003,7 @@ fn test_dont_add_floodsub_peers_to_mesh_on_join() { #[test] fn test_dont_send_px_to_old_gossipsub_peers() { - let (mut gs, _, queues, topics) = inject_nodes1() + let (mut gs, _, receivers, topics) = inject_nodes1() .peer_no(0) .topics(vec!["test".into()]) .to_subscribe(false) @@ -4956,20 +5027,17 @@ fn test_dont_send_px_to_old_gossipsub_peers() { ); //check that prune does not contain px - assert_eq!( - count_control_msgs(&queues, |_, m| match m { - RpcOut::Prune(Prune { peers: px, .. }) => !px.is_empty(), - _ => false, - }), - 0, - "Should not send px to floodsub peers" - ); + let (control_msgs, _) = count_control_msgs(receivers, |_, m| match m { + RpcOut::Prune(Prune { peers: px, .. }) => !px.is_empty(), + _ => false, + }); + assert_eq!(control_msgs, 0, "Should not send px to floodsub peers"); } #[test] fn test_dont_send_floodsub_peers_in_px() { //build mesh with one peer - let (mut gs, peers, queues, topics) = inject_nodes1() + let (mut gs, peers, receivers, topics) = inject_nodes1() .peer_no(1) .topics(vec!["test".into()]) .to_subscribe(true) @@ -4994,14 +5062,11 @@ fn test_dont_send_floodsub_peers_in_px() { ); //check that px in prune message is empty - assert_eq!( - count_control_msgs(&queues, |_, m| match m { - RpcOut::Prune(Prune { peers: px, .. }) => !px.is_empty(), - _ => false, - }), - 0, - "Should not include floodsub peers in px" - ); + let (control_msgs, _) = count_control_msgs(receivers, |_, m| match m { + RpcOut::Prune(Prune { peers: px, .. }) => !px.is_empty(), + _ => false, + }); + assert_eq!(control_msgs, 0, "Should not include floodsub peers in px"); } #[test] @@ -5088,7 +5153,7 @@ fn test_subscribe_and_graft_with_negative_score() { ))) .create_network(); - let (mut gs2, _, queues, _) = inject_nodes1().create_network(); + let (mut gs2, _, receivers, _) = inject_nodes1().create_network(); let connection_id = ConnectionId::new_unchecked(0); @@ -5105,37 +5170,41 @@ fn test_subscribe_and_graft_with_negative_score() { //subscribe to topic in gs2 gs2.subscribe(&topic).unwrap(); - let forward_messages_to_p1 = |gs1: &mut Behaviour<_, _>, _gs2: &mut Behaviour<_, _>| { - //collect messages to p1 - let messages_to_p1 = - queues - .iter() - .filter_map(|(peer_id, c)| match c.non_priority.try_recv() { - Ok(rpc) if peer_id == &p1 => Some(rpc), - _ => None, - }); - - for message in messages_to_p1 { - gs1.on_connection_handler_event( - p2, - connection_id, - HandlerEvent::Message { - rpc: proto_to_message(&message.into_protobuf()), - invalid_messages: vec![], - }, - ); + let forward_messages_to_p1 = |gs1: &mut Behaviour<_, _>, + p1: PeerId, + p2: PeerId, + connection_id: ConnectionId, + receivers: HashMap| + -> HashMap { + let new_receivers = HashMap::new(); + for (peer_id, receiver) in receivers.into_iter() { + let non_priority = receiver.non_priority.into_inner(); + match non_priority.try_recv() { + Ok(rpc) if peer_id == p1 => { + gs1.on_connection_handler_event( + p2, + connection_id, + HandlerEvent::Message { + rpc: proto_to_message(&rpc.into_protobuf()), + invalid_messages: vec![], + }, + ); + } + _ => {} + } } + new_receivers }; //forward the subscribe message - forward_messages_to_p1(&mut gs1, &mut gs2); + let receivers = forward_messages_to_p1(&mut gs1, p1, p2, connection_id, receivers); //heartbeats on both gs1.heartbeat(); gs2.heartbeat(); //forward messages again - forward_messages_to_p1(&mut gs1, &mut gs2); + forward_messages_to_p1(&mut gs1, p1, p2, connection_id, receivers); //nobody got penalized assert!(gs1.peer_score.as_ref().unwrap().0.score(&p2) >= original_score); diff --git a/beacon_node/lighthouse_network/src/gossipsub/handler.rs b/beacon_node/lighthouse_network/src/gossipsub/handler.rs index a8a980ae87e..298570955fc 100644 --- a/beacon_node/lighthouse_network/src/gossipsub/handler.rs +++ b/beacon_node/lighthouse_network/src/gossipsub/handler.rs @@ -176,12 +176,6 @@ impl Handler { } impl EnabledHandler { - #[cfg(test)] - /// For testing purposed obtain the RPCReceiver - pub fn receiver(&mut self) -> RpcReceiver { - self.send_queue.clone() - } - fn on_fully_negotiated_inbound( &mut self, (substream, peer_kind): (Framed, PeerKind), @@ -237,7 +231,7 @@ impl EnabledHandler { } // determine if we need to create the outbound stream - if !self.send_queue.is_empty() + if !self.send_queue.poll_is_empty(cx) && self.outbound_substream.is_none() && !self.outbound_substream_establishing { @@ -247,10 +241,6 @@ impl EnabledHandler { }); } - // We may need to inform the behviour if we have a dropped a message. This gets set if that - // is the case. - let mut dropped_message = None; - // process outbound stream loop { match std::mem::replace( @@ -271,10 +261,11 @@ impl EnabledHandler { } => { if Pin::new(timeout).poll(cx).is_ready() { // Inform the behaviour and end the poll. - dropped_message = Some(HandlerEvent::MessageDropped(message)); self.outbound_substream = Some(OutboundSubstreamState::WaitingOutput(substream)); - break; + return Poll::Ready(ConnectionHandlerEvent::NotifyBehaviour( + HandlerEvent::MessageDropped(message), + )); } } _ => {} // All other messages are not time-bound. @@ -348,13 +339,7 @@ impl EnabledHandler { } } - // If there was a timeout in sending a message, inform the behaviour before restarting the - // poll - if let Some(handler_event) = dropped_message { - return Poll::Ready(ConnectionHandlerEvent::NotifyBehaviour(handler_event)); - } - - // Handle inbound messages + // Handle inbound messages. loop { match std::mem::replace( &mut self.inbound_substream, @@ -419,6 +404,13 @@ impl EnabledHandler { } } + // Drop the next message in queue if it's stale. + if let Poll::Ready(Some(rpc)) = self.send_queue.poll_stale(cx) { + return Poll::Ready(ConnectionHandlerEvent::NotifyBehaviour( + HandlerEvent::MessageDropped(rpc), + )); + } + Poll::Pending } } diff --git a/beacon_node/lighthouse_network/src/gossipsub/types.rs b/beacon_node/lighthouse_network/src/gossipsub/types.rs index b7bcbf6b3ac..f77185c7c58 100644 --- a/beacon_node/lighthouse_network/src/gossipsub/types.rs +++ b/beacon_node/lighthouse_network/src/gossipsub/types.rs @@ -22,7 +22,8 @@ use crate::gossipsub::metrics::Metrics; use crate::gossipsub::TopicHash; use async_channel::{Receiver, Sender}; -use futures::Stream; +use futures::stream::Peekable; +use futures::{Future, Stream, StreamExt}; use futures_timer::Delay; use instant::Duration; use libp2p::identity::PeerId; @@ -33,7 +34,7 @@ use std::collections::BTreeSet; use std::fmt::Debug; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; -use std::task::Poll; +use std::task::{Context, Poll}; use std::{fmt, pin::Pin}; use crate::gossipsub::rpc_proto::proto; @@ -591,9 +592,10 @@ impl fmt::Display for PeerKind { pub(crate) struct RpcSender { cap: usize, len: Arc, - priority: Sender, - non_priority: Sender, - receiver: RpcReceiver, + pub(crate) priority_sender: Sender, + pub(crate) non_priority_sender: Sender, + priority_receiver: Receiver, + non_priority_receiver: Receiver, } impl RpcSender { @@ -602,29 +604,29 @@ impl RpcSender { let (priority_sender, priority_receiver) = async_channel::unbounded(); let (non_priority_sender, non_priority_receiver) = async_channel::bounded(cap / 2); let len = Arc::new(AtomicUsize::new(0)); - let receiver = RpcReceiver { - priority_len: len.clone(), - priority: priority_receiver, - non_priority: non_priority_receiver, - }; RpcSender { cap: cap / 2, len, - priority: priority_sender, - non_priority: non_priority_sender, - receiver: receiver.clone(), + priority_sender, + non_priority_sender, + priority_receiver, + non_priority_receiver, } } /// Create a new Receiver to the sender. pub(crate) fn new_receiver(&self) -> RpcReceiver { - self.receiver.clone() + RpcReceiver { + priority_len: self.len.clone(), + priority: self.priority_receiver.clone().peekable(), + non_priority: self.non_priority_receiver.clone().peekable(), + } } /// Send a `RpcOut::Graft` message to the `RpcReceiver` /// this is high priority. pub(crate) fn graft(&mut self, graft: Graft) { - self.priority + self.priority_sender .try_send(RpcOut::Graft(graft)) .expect("Channel is unbounded and should always be open"); } @@ -632,7 +634,7 @@ impl RpcSender { /// Send a `RpcOut::Prune` message to the `RpcReceiver` /// this is high priority. pub(crate) fn prune(&mut self, prune: Prune) { - self.priority + self.priority_sender .try_send(RpcOut::Prune(prune)) .expect("Channel is unbounded and should always be open"); } @@ -641,7 +643,7 @@ impl RpcSender { /// this is low priority, if the queue is full an Err is returned. #[allow(clippy::result_large_err)] pub(crate) fn ihave(&mut self, ihave: IHave) -> Result<(), RpcOut> { - self.non_priority + self.non_priority_sender .try_send(RpcOut::IHave(ihave)) .map_err(|err| err.into_inner()) } @@ -650,7 +652,7 @@ impl RpcSender { /// this is low priority, if the queue is full an Err is returned. #[allow(clippy::result_large_err)] pub(crate) fn iwant(&mut self, iwant: IWant) -> Result<(), RpcOut> { - self.non_priority + self.non_priority_sender .try_send(RpcOut::IWant(iwant)) .map_err(|err| err.into_inner()) } @@ -658,7 +660,7 @@ impl RpcSender { /// Send a `RpcOut::Subscribe` message to the `RpcReceiver` /// this is high priority. pub(crate) fn subscribe(&mut self, topic: TopicHash) { - self.priority + self.priority_sender .try_send(RpcOut::Subscribe(topic)) .expect("Channel is unbounded and should always be open"); } @@ -666,7 +668,7 @@ impl RpcSender { /// Send a `RpcOut::Unsubscribe` message to the `RpcReceiver` /// this is high priority. pub(crate) fn unsubscribe(&mut self, topic: TopicHash) { - self.priority + self.priority_sender .try_send(RpcOut::Unsubscribe(topic)) .expect("Channel is unbounded and should always be open"); } @@ -682,7 +684,7 @@ impl RpcSender { if self.len.load(Ordering::Relaxed) >= self.cap { return Err(()); } - self.priority + self.priority_sender .try_send(RpcOut::Publish { message: message.clone(), timeout: Delay::new(timeout), @@ -705,7 +707,7 @@ impl RpcSender { timeout: Duration, metrics: Option<&mut Metrics>, ) -> Result<(), ()> { - self.non_priority + self.non_priority_sender .try_send(RpcOut::Forward { message: message.clone(), timeout: Delay::new(timeout), @@ -726,25 +728,73 @@ impl RpcSender { /// Returns the current size of the non-priority queue. pub(crate) fn non_priority_len(&self) -> usize { - self.non_priority.len() + self.non_priority_sender.len() } } /// `RpcOut` sender that is priority aware. -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct RpcReceiver { /// The maximum length of the priority queue. - priority_len: Arc, + pub(crate) priority_len: Arc, /// The priority queue receiver. - pub(crate) priority: Receiver, + pub(crate) priority: Peekable>, /// The non priority queue receiver. - pub(crate) non_priority: Receiver, + pub(crate) non_priority: Peekable>, } impl RpcReceiver { - /// Check if both queues are empty. - pub(crate) fn is_empty(&self) -> bool { - self.priority.is_empty() && self.non_priority.is_empty() + // Peek the next message in the queues and return it if its timeout has elapsed. + // Returns `None` if there aren't any more messages on the stream or none is stale. + pub(crate) fn poll_stale(&mut self, cx: &mut Context<'_>) -> Poll> { + // Peek priority queue. + let priority = match Pin::new(&mut self.priority).poll_peek_mut(cx) { + Poll::Ready(Some(RpcOut::Publish { + message: _, + ref mut timeout, + })) => { + if Pin::new(timeout).poll(cx).is_ready() { + // Return the message. + let dropped = futures::ready!(self.priority.poll_next_unpin(cx)) + .expect("There should be a message"); + return Poll::Ready(Some(dropped)); + } + Poll::Ready(None) + } + poll => poll, + }; + + let non_priority = match Pin::new(&mut self.non_priority).poll_peek_mut(cx) { + Poll::Ready(Some(RpcOut::Forward { + message: _, + ref mut timeout, + })) => { + if Pin::new(timeout).poll(cx).is_ready() { + // Return the message. + let dropped = futures::ready!(self.non_priority.poll_next_unpin(cx)) + .expect("There should be a message"); + return Poll::Ready(Some(dropped)); + } + Poll::Ready(None) + } + poll => poll, + }; + + match (priority, non_priority) { + (Poll::Ready(None), Poll::Ready(None)) => Poll::Ready(None), + _ => Poll::Pending, + } + } + + /// Poll queues and return true if both are empty. + pub(crate) fn poll_is_empty(&mut self, cx: &mut Context<'_>) -> bool { + matches!( + ( + Pin::new(&mut self.priority).poll_peek(cx), + Pin::new(&mut self.non_priority).poll_peek(cx), + ), + (Poll::Ready(None), Poll::Ready(None)) + ) } } From 7c236251933dc87760d8c1ed052c7dbac67d497b Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 15 Feb 2024 15:18:23 +1100 Subject: [PATCH 12/24] Quieten gossip republish logs (#5235) * Quieten gossip republish logs --- .../lighthouse_network/src/service/mod.rs | 37 ++++++++++++++----- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/beacon_node/lighthouse_network/src/service/mod.rs b/beacon_node/lighthouse_network/src/service/mod.rs index 879f13ad7c8..401e43a53ff 100644 --- a/beacon_node/lighthouse_network/src/service/mod.rs +++ b/beacon_node/lighthouse_network/src/service/mod.rs @@ -1226,22 +1226,39 @@ impl Network { .publish(Topic::from(topic.clone()), data) { Ok(_) => { - warn!(self.log, "Gossip message published on retry"; "topic" => topic_str); - if let Some(v) = metrics::get_int_counter( + debug!( + self.log, + "Gossip message published on retry"; + "topic" => topic_str + ); + metrics::inc_counter_vec( &metrics::GOSSIP_LATE_PUBLISH_PER_TOPIC_KIND, &[topic_str], - ) { - v.inc() - }; + ); + } + Err(PublishError::Duplicate) => { + debug!( + self.log, + "Gossip message publish ignored on retry"; + "reason" => "duplicate", + "topic" => topic_str + ); + metrics::inc_counter_vec( + &metrics::GOSSIP_FAILED_LATE_PUBLISH_PER_TOPIC_KIND, + &[topic_str], + ); } Err(e) => { - warn!(self.log, "Gossip message publish failed on retry"; "topic" => topic_str, "error" => %e); - if let Some(v) = metrics::get_int_counter( + warn!( + self.log, + "Gossip message publish failed on retry"; + "topic" => topic_str, + "error" => %e + ); + metrics::inc_counter_vec( &metrics::GOSSIP_FAILED_LATE_PUBLISH_PER_TOPIC_KIND, &[topic_str], - ) { - v.inc() - }; + ); } } } From 0e819fa78543ab9d8e4d626ed5df357915535a2e Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 15 Feb 2024 23:23:51 +1100 Subject: [PATCH 13/24] Schedule Deneb on mainnet (#5233) * Schedule Deneb on mainnet * Fix trusted setup roundtrip test * Fix BN CLI tests for insecure genesis sync --- .../mainnet/config.yaml | 68 ++++++++++++++----- common/eth2_network_config/src/lib.rs | 4 +- consensus/types/presets/mainnet/deneb.yaml | 2 +- consensus/types/src/chain_spec.rs | 2 +- lighthouse/tests/beacon_node.rs | 35 ++++++++-- 5 files changed, 84 insertions(+), 27 deletions(-) diff --git a/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml b/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml index 019eda43243..b29ecfc6d38 100644 --- a/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/mainnet/config.yaml @@ -1,9 +1,15 @@ # Mainnet config # Extends the mainnet preset -CONFIG_NAME: 'mainnet' PRESET_BASE: 'mainnet' +# Free-form short name of the network that this configuration applies to - known +# canonical network names include: +# * 'mainnet' - there can be only one +# * 'prater' - testnet +# Must match the regex: [a-z0-9\-] +CONFIG_NAME: 'mainnet' + # Transition # --------------------------------------------------------------- # Estimated on Sept 15, 2022 @@ -12,6 +18,8 @@ TERMINAL_TOTAL_DIFFICULTY: 58750000000000000000000 TERMINAL_BLOCK_HASH: 0x0000000000000000000000000000000000000000000000000000000000000000 TERMINAL_BLOCK_HASH_ACTIVATION_EPOCH: 18446744073709551615 + + # Genesis # --------------------------------------------------------------- # `2**14` (= 16,384) @@ -32,22 +40,16 @@ GENESIS_DELAY: 604800 # Altair ALTAIR_FORK_VERSION: 0x01000000 -ALTAIR_FORK_EPOCH: 74240 -# Merge +ALTAIR_FORK_EPOCH: 74240 # Oct 27, 2021, 10:56:23am UTC +# Bellatrix BELLATRIX_FORK_VERSION: 0x02000000 -BELLATRIX_FORK_EPOCH: 144896 # Sept 6, 2022, 11:34:47am UTC +BELLATRIX_FORK_EPOCH: 144896 # Sept 6, 2022, 11:34:47am UTC # Capella CAPELLA_FORK_VERSION: 0x03000000 -CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC +CAPELLA_FORK_EPOCH: 194048 # April 12, 2023, 10:27:35pm UTC # Deneb DENEB_FORK_VERSION: 0x04000000 -DENEB_FORK_EPOCH: 18446744073709551615 -# Sharding -SHARDING_FORK_VERSION: 0x03000000 -SHARDING_FORK_EPOCH: 18446744073709551615 - -# TBD, 2**32 is a placeholder. Merge transition approach is in active R&D. -TRANSITION_TOTAL_DIFFICULTY: 4294967296 +DENEB_FORK_EPOCH: 269568 # March 13, 2024, 01:55:35pm UTC # Time parameters @@ -74,16 +76,22 @@ INACTIVITY_SCORE_RECOVERY_RATE: 16 EJECTION_BALANCE: 16000000000 # 2**2 (= 4) MIN_PER_EPOCH_CHURN_LIMIT: 4 -# 2**3 (= 8) -MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8 # 2**16 (= 65,536) CHURN_LIMIT_QUOTIENT: 65536 - +# [New in Deneb:EIP7514] 2**3 (= 8) +MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8 # Fork choice # --------------------------------------------------------------- # 40% PROPOSER_SCORE_BOOST: 40 +# 20% +REORG_HEAD_WEIGHT_THRESHOLD: 20 +# 160% +REORG_PARENT_WEIGHT_THRESHOLD: 160 +# `2` epochs +REORG_MAX_EPOCHS_SINCE_FINALIZATION: 2 + # Deposit contract # --------------------------------------------------------------- @@ -92,17 +100,43 @@ DEPOSIT_CHAIN_ID: 1 DEPOSIT_NETWORK_ID: 1 DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa -# Network + +# Networking # --------------------------------------------------------------- -SUBNETS_PER_NODE: 2 +# `10 * 2**20` (= 10485760, 10 MiB) GOSSIP_MAX_SIZE: 10485760 +# `2**10` (= 1024) +MAX_REQUEST_BLOCKS: 1024 +# `2**8` (= 256) +EPOCHS_PER_SUBNET_SUBSCRIPTION: 256 +# `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months) MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024 +# `10 * 2**20` (=10485760, 10 MiB) MAX_CHUNK_SIZE: 10485760 +# 5s TTFB_TIMEOUT: 5 +# 10s RESP_TIMEOUT: 10 +ATTESTATION_PROPAGATION_SLOT_RANGE: 32 +# 500ms +MAXIMUM_GOSSIP_CLOCK_DISPARITY: 500 MESSAGE_DOMAIN_INVALID_SNAPPY: 0x00000000 MESSAGE_DOMAIN_VALID_SNAPPY: 0x01000000 +# 2 subnets per node +SUBNETS_PER_NODE: 2 +# 2**8 (= 64) ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 +# ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS ATTESTATION_SUBNET_PREFIX_BITS: 6 ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3 + +# Deneb +# `2**7` (=128) +MAX_REQUEST_BLOCKS_DENEB: 128 +# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK +MAX_REQUEST_BLOB_SIDECARS: 768 +# `2**12` (= 4096 epochs, ~18 days) +MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096 +# `6` +BLOB_SIDECAR_SUBNET_COUNT: 6 diff --git a/common/eth2_network_config/src/lib.rs b/common/eth2_network_config/src/lib.rs index 565b8d78902..a76a8320aa8 100644 --- a/common/eth2_network_config/src/lib.rs +++ b/common/eth2_network_config/src/lib.rs @@ -583,6 +583,8 @@ mod tests { } else { GenesisStateSource::Unknown }; + // With Deneb enabled by default we must set a trusted setup here. + let kzg_trusted_setup = get_trusted_setup_from_config(&config).unwrap(); let testnet = Eth2NetworkConfig { deposit_contract_deploy_block, @@ -593,7 +595,7 @@ mod tests { .map(Encode::as_ssz_bytes) .map(Into::into), config, - kzg_trusted_setup: None, + kzg_trusted_setup: Some(kzg_trusted_setup), }; testnet diff --git a/consensus/types/presets/mainnet/deneb.yaml b/consensus/types/presets/mainnet/deneb.yaml index 6d2fb4abde9..0f56b8bdfac 100644 --- a/consensus/types/presets/mainnet/deneb.yaml +++ b/consensus/types/presets/mainnet/deneb.yaml @@ -8,5 +8,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096 MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096 # `uint64(6)` MAX_BLOBS_PER_BLOCK: 6 -# `floorlog2(BLOB_KZG_COMMITMENTS_GINDEX) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17 +# `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17 diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index a182c0f98d6..c32c67fa330 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -681,7 +681,7 @@ impl ChainSpec { * Deneb hard fork params */ deneb_fork_version: [0x04, 0x00, 0x00, 0x00], - deneb_fork_epoch: None, + deneb_fork_epoch: Some(Epoch::new(269568)), /* * Network specific diff --git a/lighthouse/tests/beacon_node.rs b/lighthouse/tests/beacon_node.rs index f97f17a6677..94996eb1a26 100644 --- a/lighthouse/tests/beacon_node.rs +++ b/lighthouse/tests/beacon_node.rs @@ -55,6 +55,12 @@ impl CommandLineTest { } fn run_with_zero_port(&mut self) -> CompletedTest { + // Required since Deneb was enabled on mainnet. + self.cmd.arg("--allow-insecure-genesis-sync"); + self.run_with_zero_port_and_no_genesis_sync() + } + + fn run_with_zero_port_and_no_genesis_sync(&mut self) -> CompletedTest { self.cmd.arg("-z"); self.run() } @@ -93,16 +99,16 @@ fn staking_flag() { } #[test] -fn allow_insecure_genesis_sync() { - CommandLineTest::new() - .run_with_zero_port() - .with_config(|config| { - assert_eq!(config.allow_insecure_genesis_sync, false); - }); +#[should_panic] +fn allow_insecure_genesis_sync_default() { + CommandLineTest::new().run_with_zero_port_and_no_genesis_sync(); +} +#[test] +fn allow_insecure_genesis_sync_enabled() { CommandLineTest::new() .flag("allow-insecure-genesis-sync", None) - .run_with_zero_port() + .run_with_zero_port_and_no_genesis_sync() .with_config(|config| { assert_eq!(config.allow_insecure_genesis_sync, true); }); @@ -851,6 +857,7 @@ fn network_port_flag_over_ipv4() { let port = 0; CommandLineTest::new() .flag("port", Some(port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -867,6 +874,7 @@ fn network_port_flag_over_ipv4() { let port = 9000; CommandLineTest::new() .flag("port", Some(port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -886,6 +894,7 @@ fn network_port_flag_over_ipv6() { CommandLineTest::new() .flag("listen-address", Some("::1")) .flag("port", Some(port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -903,6 +912,7 @@ fn network_port_flag_over_ipv6() { CommandLineTest::new() .flag("listen-address", Some("::1")) .flag("port", Some(port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -925,6 +935,7 @@ fn network_port_flag_over_ipv4_and_ipv6() { .flag("listen-address", Some("::1")) .flag("port", Some(port.to_string().as_str())) .flag("port6", Some(port6.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -954,6 +965,7 @@ fn network_port_flag_over_ipv4_and_ipv6() { .flag("listen-address", Some("::1")) .flag("port", Some(port.to_string().as_str())) .flag("port6", Some(port6.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -983,6 +995,7 @@ fn network_port_and_discovery_port_flags_over_ipv4() { CommandLineTest::new() .flag("port", Some(tcp4_port.to_string().as_str())) .flag("discovery-port", Some(disc4_port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -1003,6 +1016,7 @@ fn network_port_and_discovery_port_flags_over_ipv6() { .flag("listen-address", Some("::1")) .flag("port", Some(tcp6_port.to_string().as_str())) .flag("discovery-port", Some(disc6_port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -1028,6 +1042,7 @@ fn network_port_and_discovery_port_flags_over_ipv4_and_ipv6() { .flag("discovery-port", Some(disc4_port.to_string().as_str())) .flag("port6", Some(tcp6_port.to_string().as_str())) .flag("discovery-port6", Some(disc6_port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -1067,6 +1082,7 @@ fn network_port_discovery_quic_port_flags_over_ipv4_and_ipv6() { .flag("port6", Some(tcp6_port.to_string().as_str())) .flag("discovery-port6", Some(disc6_port.to_string().as_str())) .flag("quic-port6", Some(quic6_port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -1293,6 +1309,7 @@ fn enr_match_flag_over_ipv4() { .flag("listen-address", Some("127.0.0.2")) .flag("discovery-port", Some(udp4_port.to_string().as_str())) .flag("port", Some(tcp4_port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -1324,6 +1341,7 @@ fn enr_match_flag_over_ipv6() { .flag("listen-address", Some(ADDR)) .flag("discovery-port", Some(udp6_port.to_string().as_str())) .flag("port", Some(tcp6_port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -1364,6 +1382,7 @@ fn enr_match_flag_over_ipv4_and_ipv6() { .flag("listen-address", Some(IPV6_ADDR)) .flag("discovery-port6", Some(udp6_port.to_string().as_str())) .flag("port6", Some(tcp6_port.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| { assert_eq!( @@ -1490,6 +1509,7 @@ fn http_port_flag() { .flag("http", None) .flag("http-port", Some(port1.to_string().as_str())) .flag("port", Some(port2.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| assert_eq!(config.http_api.listen_port, port1)); } @@ -1647,6 +1667,7 @@ fn metrics_port_flag() { .flag("metrics", None) .flag("metrics-port", Some(port1.to_string().as_str())) .flag("port", Some(port2.to_string().as_str())) + .flag("allow-insecure-genesis-sync", None) .run() .with_config(|config| assert_eq!(config.http_metrics.listen_port, port1)); } From 49536ff103c2ec151950f41b61ed34ab59251076 Mon Sep 17 00:00:00 2001 From: Age Manning Date: Thu, 15 Feb 2024 23:23:58 +1100 Subject: [PATCH 14/24] Add `distributed` flag to VC to enable support for DVT (#4867) * Initial flag building * Update validator_client/src/cli.rs Co-authored-by: Abhishek Kumar <43061995+xenowits@users.noreply.github.com> * Merge latest unstable * Per slot aggregates * One slot lookahead for sync committee aggregates * Update validator_client/src/duties_service.rs Co-authored-by: Abhishek Kumar <43061995+xenowits@users.noreply.github.com> * Rename selection_look_ahead * Merge branch 'unstable' into vc-distributed * Merge remote-tracking branch 'origin/unstable' into vc-distributed * Update CLI text --- book/src/help_vc.md | 3 + validator_client/Cargo.toml | 2 +- validator_client/src/cli.rs | 6 + validator_client/src/config.rs | 7 + validator_client/src/duties_service.rs | 22 +- validator_client/src/duties_service/sync.rs | 251 ++++++++++-------- validator_client/src/lib.rs | 5 +- .../src/sync_committee_service.rs | 4 +- 8 files changed, 173 insertions(+), 127 deletions(-) diff --git a/book/src/help_vc.md b/book/src/help_vc.md index 02819804315..3d2519aac57 100644 --- a/book/src/help_vc.md +++ b/book/src/help_vc.md @@ -26,6 +26,9 @@ FLAGS: but is only safe if slashing protection is enabled on the remote signer and is implemented correctly. DO NOT ENABLE THIS FLAG UNLESS YOU ARE CERTAIN THAT SLASHING PROTECTION IS ENABLED ON THE REMOTE SIGNER. YOU WILL GET SLASHED IF YOU USE THIS FLAG WITHOUT ENABLING WEB3SIGNER'S SLASHING PROTECTION. + --distributed + Enables functionality required for running the validator in a distributed validator cluster. + --enable-doppelganger-protection If this flag is set, Lighthouse will delay startup for three epochs and monitor for messages on the network by any of the validators managed by this client. This will result in three (possibly four) epochs worth of diff --git a/validator_client/Cargo.toml b/validator_client/Cargo.toml index 90a82b7e3b2..8e587c6155f 100644 --- a/validator_client/Cargo.toml +++ b/validator_client/Cargo.toml @@ -10,6 +10,7 @@ path = "src/lib.rs" [dev-dependencies] tokio = { workspace = true } +itertools = { workspace = true } [dependencies] tree_hash = { workspace = true } @@ -51,7 +52,6 @@ ring = { workspace = true } rand = { workspace = true, features = ["small_rng"] } lighthouse_metrics = { workspace = true } lazy_static = { workspace = true } -itertools = { workspace = true } monitoring_api = { workspace = true } sensitive_url = { workspace = true } task_executor = { workspace = true } diff --git a/validator_client/src/cli.rs b/validator_client/src/cli.rs index e4e5a2f1307..16a265212e5 100644 --- a/validator_client/src/cli.rs +++ b/validator_client/src/cli.rs @@ -145,6 +145,12 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { future.") .takes_value(false) ) + .arg( + Arg::with_name("distributed") + .long("distributed") + .help("Enables functionality required for running the validator in a distributed validator cluster.") + .takes_value(false) + ) /* REST API related arguments */ .arg( Arg::with_name("http") diff --git a/validator_client/src/config.rs b/validator_client/src/config.rs index 7ac9e3e3bc7..ae59829a3e6 100644 --- a/validator_client/src/config.rs +++ b/validator_client/src/config.rs @@ -84,6 +84,8 @@ pub struct Config { pub builder_boost_factor: Option, /// If true, Lighthouse will prefer builder proposals, if available. pub prefer_builder_proposals: bool, + /// Whether we are running with distributed network support. + pub distributed: bool, pub web3_signer_keep_alive_timeout: Option, pub web3_signer_max_idle_connections: Option, } @@ -130,6 +132,7 @@ impl Default for Config { produce_block_v3: false, builder_boost_factor: None, prefer_builder_proposals: false, + distributed: false, web3_signer_keep_alive_timeout: Some(Duration::from_secs(90)), web3_signer_max_idle_connections: None, } @@ -233,6 +236,10 @@ impl Config { config.beacon_nodes_tls_certs = Some(tls_certs.split(',').map(PathBuf::from).collect()); } + if cli_args.is_present("distributed") { + config.distributed = true; + } + if cli_args.is_present("disable-run-on-all") { warn!( log, diff --git a/validator_client/src/duties_service.rs b/validator_client/src/duties_service.rs index 26747f81110..290803e257a 100644 --- a/validator_client/src/duties_service.rs +++ b/validator_client/src/duties_service.rs @@ -6,7 +6,7 @@ //! The `DutiesService` is also responsible for sending events to the `BlockService` which trigger //! block production. -mod sync; +pub mod sync; use crate::beacon_node_fallback::{ApiTopic, BeaconNodeFallback, OfflineOnFailure, RequireSynced}; use crate::http_metrics::metrics::{get_int_gauge, set_int_gauge, ATTESTATION_DUTY}; @@ -42,6 +42,9 @@ const HISTORICAL_DUTIES_EPOCHS: u64 = 2; /// At start-up selection proofs will be computed with less lookahead out of necessity. const SELECTION_PROOF_SLOT_LOOKAHEAD: u64 = 8; +/// The attestation selection proof lookahead for those running with the --distributed flag. +const SELECTION_PROOF_SLOT_LOOKAHEAD_DVT: u64 = 1; + /// Fraction of a slot at which selection proof signing should happen (2 means half way). const SELECTION_PROOF_SCHEDULE_DENOM: u32 = 2; @@ -211,16 +214,21 @@ pub struct DutiesService { /// proposals for any validators which are not registered locally. pub proposers: RwLock, /// Map from validator index to sync committee duties. - pub sync_duties: SyncDutiesMap, + pub sync_duties: SyncDutiesMap, /// Provides the canonical list of locally-managed validators. pub validator_store: Arc>, /// Tracks the current slot. pub slot_clock: T, /// Provides HTTP access to remote beacon nodes. pub beacon_nodes: Arc>, - pub enable_high_validator_count_metrics: bool, + /// The runtime for spawning tasks. pub context: RuntimeContext, + /// The current chain spec. pub spec: ChainSpec, + //// Whether we permit large validator counts in the metrics. + pub enable_high_validator_count_metrics: bool, + /// If this validator is running in distributed mode. + pub distributed: bool, } impl DutiesService { @@ -997,7 +1005,13 @@ async fn fill_in_selection_proofs( continue; }; - let lookahead_slot = current_slot + SELECTION_PROOF_SLOT_LOOKAHEAD; + let selection_lookahead = if duties_service.distributed { + SELECTION_PROOF_SLOT_LOOKAHEAD_DVT + } else { + SELECTION_PROOF_SLOT_LOOKAHEAD + }; + + let lookahead_slot = current_slot + selection_lookahead; let mut relevant_duties = duties_by_slot.split_off(&lookahead_slot); std::mem::swap(&mut relevant_duties, &mut duties_by_slot); diff --git a/validator_client/src/duties_service/sync.rs b/validator_client/src/duties_service/sync.rs index de42fa587ef..3618b47146f 100644 --- a/validator_client/src/duties_service/sync.rs +++ b/validator_client/src/duties_service/sync.rs @@ -7,18 +7,18 @@ use crate::{ }; use futures::future::join_all; -use itertools::Itertools; use parking_lot::{MappedRwLockReadGuard, RwLock, RwLockReadGuard, RwLockWriteGuard}; use slog::{crit, debug, info, warn}; use slot_clock::SlotClock; use std::collections::{HashMap, HashSet}; +use std::marker::PhantomData; use std::sync::Arc; -use types::{ - ChainSpec, Epoch, EthSpec, PublicKeyBytes, Slot, SyncDuty, SyncSelectionProof, SyncSubnetId, -}; +use types::{ChainSpec, EthSpec, PublicKeyBytes, Slot, SyncDuty, SyncSelectionProof, SyncSubnetId}; -/// Number of epochs in advance to compute selection proofs. +/// Number of epochs in advance to compute selection proofs when not in `distributed` mode. pub const AGGREGATION_PRE_COMPUTE_EPOCHS: u64 = 2; +/// Number of slots in advance to compute selection proofs when in `distributed` mode. +pub const AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED: u64 = 1; /// Top-level data-structure containing sync duty information. /// @@ -32,9 +32,12 @@ pub const AGGREGATION_PRE_COMPUTE_EPOCHS: u64 = 2; /// 2. One-at-a-time locking. For the innermost locks on the aggregator duties, all of the functions /// in this file take care to only lock one validator at a time. We never hold a lock while /// trying to obtain another one (hence no lock ordering issues). -pub struct SyncDutiesMap { +pub struct SyncDutiesMap { /// Map from sync committee period to duties for members of that sync committee. committees: RwLock>, + /// Whether we are in `distributed` mode and using reduced lookahead for aggregate pre-compute. + distributed: bool, + _phantom: PhantomData, } /// Duties for a single sync committee period. @@ -59,8 +62,8 @@ pub struct ValidatorDuties { /// Aggregator duties for a single validator. pub struct AggregatorDuties { - /// The epoch up to which aggregation proofs have already been computed (inclusive). - pre_compute_epoch: RwLock>, + /// The slot up to which aggregation proofs have already been computed (inclusive). + pre_compute_slot: RwLock>, /// Map from slot & subnet ID to proof that this validator is an aggregator. /// /// The slot is the slot at which the signed contribution and proof should be broadcast, @@ -82,15 +85,15 @@ pub struct SlotDuties { pub aggregators: HashMap>, } -impl Default for SyncDutiesMap { - fn default() -> Self { +impl SyncDutiesMap { + pub fn new(distributed: bool) -> Self { Self { committees: RwLock::new(HashMap::new()), + distributed, + _phantom: PhantomData, } } -} -impl SyncDutiesMap { /// Check if duties are already known for all of the given validators for `committee_period`. fn all_duties_known(&self, committee_period: u64, validator_indices: &[u64]) -> bool { self.committees @@ -104,22 +107,34 @@ impl SyncDutiesMap { }) } + /// Number of slots in advance to compute selection proofs + fn aggregation_pre_compute_slots(&self) -> u64 { + if self.distributed { + AGGREGATION_PRE_COMPUTE_SLOTS_DISTRIBUTED + } else { + E::slots_per_epoch() * AGGREGATION_PRE_COMPUTE_EPOCHS + } + } + /// Prepare for pre-computation of selection proofs for `committee_period`. /// - /// Return the epoch up to which proofs should be pre-computed, as well as a vec of - /// `(previous_pre_compute_epoch, sync_duty)` pairs for all validators which need to have proofs + /// Return the slot up to which proofs should be pre-computed, as well as a vec of + /// `(previous_pre_compute_slot, sync_duty)` pairs for all validators which need to have proofs /// computed. See `fill_in_aggregation_proofs` for the actual calculation. fn prepare_for_aggregator_pre_compute( &self, committee_period: u64, - current_epoch: Epoch, + current_slot: Slot, spec: &ChainSpec, - ) -> (Epoch, Vec<(Epoch, SyncDuty)>) { - let default_start_epoch = - std::cmp::max(current_epoch, first_epoch_of_period(committee_period, spec)); - let pre_compute_epoch = std::cmp::min( - current_epoch + AGGREGATION_PRE_COMPUTE_EPOCHS, - last_epoch_of_period(committee_period, spec), + ) -> (Slot, Vec<(Slot, SyncDuty)>) { + let default_start_slot = std::cmp::max( + current_slot, + first_slot_of_period::(committee_period, spec), + ); + let pre_compute_lookahead_slots = self.aggregation_pre_compute_slots(); + let pre_compute_slot = std::cmp::min( + current_slot + pre_compute_lookahead_slots, + last_slot_of_period::(committee_period, spec), ); let pre_compute_duties = self.committees.read().get(&committee_period).map_or_else( @@ -130,18 +145,18 @@ impl SyncDutiesMap { .values() .filter_map(|maybe_duty| { let duty = maybe_duty.as_ref()?; - let old_pre_compute_epoch = duty + let old_pre_compute_slot = duty .aggregation_duties - .pre_compute_epoch + .pre_compute_slot .write() - .replace(pre_compute_epoch); + .replace(pre_compute_slot); - match old_pre_compute_epoch { + match old_pre_compute_slot { // No proofs pre-computed previously, compute all from the start of - // the period or the current epoch (whichever is later). - None => Some((default_start_epoch, duty.duty.clone())), + // the period or the current slot (whichever is later). + None => Some((default_start_slot, duty.duty.clone())), // Proofs computed up to `prev`, start from the subsequent epoch. - Some(prev) if prev < pre_compute_epoch => { + Some(prev) if prev < pre_compute_slot => { Some((prev + 1, duty.duty.clone())) } // Proofs already known, no need to compute. @@ -151,7 +166,7 @@ impl SyncDutiesMap { .collect() }, ); - (pre_compute_epoch, pre_compute_duties) + (pre_compute_slot, pre_compute_duties) } fn get_or_create_committee_duties<'a, 'b>( @@ -176,7 +191,7 @@ impl SyncDutiesMap { /// Get duties for all validators for the given `wall_clock_slot`. /// /// This is the entry-point for the sync committee service. - pub fn get_duties_for_slot( + pub fn get_duties_for_slot( &self, wall_clock_slot: Slot, spec: &ChainSpec, @@ -253,7 +268,7 @@ impl ValidatorDuties { Self { duty, aggregation_duties: AggregatorDuties { - pre_compute_epoch: RwLock::new(None), + pre_compute_slot: RwLock::new(None), proofs: RwLock::new(HashMap::new()), }, } @@ -265,12 +280,12 @@ fn epoch_offset(spec: &ChainSpec) -> u64 { spec.epochs_per_sync_committee_period.as_u64() / 2 } -fn first_epoch_of_period(sync_committee_period: u64, spec: &ChainSpec) -> Epoch { - spec.epochs_per_sync_committee_period * sync_committee_period +fn first_slot_of_period(sync_committee_period: u64, spec: &ChainSpec) -> Slot { + (spec.epochs_per_sync_committee_period * sync_committee_period).start_slot(E::slots_per_epoch()) } -fn last_epoch_of_period(sync_committee_period: u64, spec: &ChainSpec) -> Epoch { - first_epoch_of_period(sync_committee_period + 1, spec) - 1 +fn last_slot_of_period(sync_committee_period: u64, spec: &ChainSpec) -> Slot { + first_slot_of_period::(sync_committee_period + 1, spec) - 1 } pub async fn poll_sync_committee_duties( @@ -278,11 +293,11 @@ pub async fn poll_sync_committee_duties( ) -> Result<(), Error> { let sync_duties = &duties_service.sync_duties; let spec = &duties_service.spec; - let current_epoch = duties_service + let current_slot = duties_service .slot_clock .now() - .ok_or(Error::UnableToReadSlotClock)? - .epoch(E::slots_per_epoch()); + .ok_or(Error::UnableToReadSlotClock)?; + let current_epoch = current_slot.epoch(E::slots_per_epoch()); // If the Altair fork is yet to be activated, do not attempt to poll for duties. if spec @@ -330,8 +345,8 @@ pub async fn poll_sync_committee_duties( } // Pre-compute aggregator selection proofs for the current period. - let (current_pre_compute_epoch, new_pre_compute_duties) = sync_duties - .prepare_for_aggregator_pre_compute(current_sync_committee_period, current_epoch, spec); + let (current_pre_compute_slot, new_pre_compute_duties) = sync_duties + .prepare_for_aggregator_pre_compute(current_sync_committee_period, current_slot, spec); if !new_pre_compute_duties.is_empty() { let sub_duties_service = duties_service.clone(); @@ -341,8 +356,8 @@ pub async fn poll_sync_committee_duties( sub_duties_service, &new_pre_compute_duties, current_sync_committee_period, - current_epoch, - current_pre_compute_epoch, + current_slot, + current_pre_compute_slot, ) .await }, @@ -368,11 +383,14 @@ pub async fn poll_sync_committee_duties( } // Pre-compute aggregator selection proofs for the next period. - if (current_epoch + AGGREGATION_PRE_COMPUTE_EPOCHS).sync_committee_period(spec)? + let aggregate_pre_compute_lookahead_slots = sync_duties.aggregation_pre_compute_slots(); + if (current_slot + aggregate_pre_compute_lookahead_slots) + .epoch(E::slots_per_epoch()) + .sync_committee_period(spec)? == next_sync_committee_period { - let (pre_compute_epoch, new_pre_compute_duties) = sync_duties - .prepare_for_aggregator_pre_compute(next_sync_committee_period, current_epoch, spec); + let (pre_compute_slot, new_pre_compute_duties) = sync_duties + .prepare_for_aggregator_pre_compute(next_sync_committee_period, current_slot, spec); if !new_pre_compute_duties.is_empty() { let sub_duties_service = duties_service.clone(); @@ -382,8 +400,8 @@ pub async fn poll_sync_committee_duties( sub_duties_service, &new_pre_compute_duties, next_sync_committee_period, - current_epoch, - pre_compute_epoch, + current_slot, + pre_compute_slot, ) .await }, @@ -495,10 +513,10 @@ pub async fn poll_sync_committee_duties_for_period( duties_service: Arc>, - pre_compute_duties: &[(Epoch, SyncDuty)], + pre_compute_duties: &[(Slot, SyncDuty)], sync_committee_period: u64, - current_epoch: Epoch, - pre_compute_epoch: Epoch, + current_slot: Slot, + pre_compute_slot: Slot, ) { let log = duties_service.context.log(); @@ -506,16 +524,16 @@ pub async fn fill_in_aggregation_proofs( log, "Calculating sync selection proofs"; "period" => sync_committee_period, - "current_epoch" => current_epoch, - "pre_compute_epoch" => pre_compute_epoch + "current_slot" => current_slot, + "pre_compute_slot" => pre_compute_slot ); - // Generate selection proofs for each validator at each slot, one epoch at a time. - for epoch in (current_epoch.as_u64()..=pre_compute_epoch.as_u64()).map(Epoch::new) { + // Generate selection proofs for each validator at each slot, one slot at a time. + for slot in (current_slot.as_u64()..=pre_compute_slot.as_u64()).map(Slot::new) { let mut validator_proofs = vec![]; - for (validator_start_epoch, duty) in pre_compute_duties { - // Proofs are already known at this epoch for this validator. - if epoch < *validator_start_epoch { + for (validator_start_slot, duty) in pre_compute_duties { + // Proofs are already known at this slot for this validator. + if slot < *validator_start_slot { continue; } @@ -533,67 +551,64 @@ pub async fn fill_in_aggregation_proofs( // Create futures to produce proofs. let duties_service_ref = &duties_service; - let futures = epoch - .slot_iter(E::slots_per_epoch()) - .cartesian_product(&subnet_ids) - .map(|(duty_slot, subnet_id)| async move { - // Construct proof for prior slot. - let slot = duty_slot - 1; - - let proof = match duties_service_ref - .validator_store - .produce_sync_selection_proof(&duty.pubkey, slot, *subnet_id) - .await - { - Ok(proof) => proof, - Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { - // A pubkey can be missing when a validator was recently - // removed via the API. - debug!( - log, - "Missing pubkey for sync selection proof"; - "pubkey" => ?pubkey, - "pubkey" => ?duty.pubkey, - "slot" => slot, - ); - return None; - } - Err(e) => { - warn!( - log, - "Unable to sign selection proof"; - "error" => ?e, - "pubkey" => ?duty.pubkey, - "slot" => slot, - ); - return None; - } - }; - - match proof.is_aggregator::() { - Ok(true) => { - debug!( - log, - "Validator is sync aggregator"; - "validator_index" => duty.validator_index, - "slot" => slot, - "subnet_id" => %subnet_id, - ); - Some(((slot, *subnet_id), proof)) - } - Ok(false) => None, - Err(e) => { - warn!( - log, - "Error determining is_aggregator"; - "pubkey" => ?duty.pubkey, - "slot" => slot, - "error" => ?e, - ); - None - } + let futures = subnet_ids.iter().map(|subnet_id| async move { + // Construct proof for prior slot. + let proof_slot = slot - 1; + + let proof = match duties_service_ref + .validator_store + .produce_sync_selection_proof(&duty.pubkey, proof_slot, *subnet_id) + .await + { + Ok(proof) => proof, + Err(ValidatorStoreError::UnknownPubkey(pubkey)) => { + // A pubkey can be missing when a validator was recently + // removed via the API. + debug!( + log, + "Missing pubkey for sync selection proof"; + "pubkey" => ?pubkey, + "pubkey" => ?duty.pubkey, + "slot" => proof_slot, + ); + return None; + } + Err(e) => { + warn!( + log, + "Unable to sign selection proof"; + "error" => ?e, + "pubkey" => ?duty.pubkey, + "slot" => proof_slot, + ); + return None; } - }); + }; + + match proof.is_aggregator::() { + Ok(true) => { + debug!( + log, + "Validator is sync aggregator"; + "validator_index" => duty.validator_index, + "slot" => proof_slot, + "subnet_id" => %subnet_id, + ); + Some(((proof_slot, *subnet_id), proof)) + } + Ok(false) => None, + Err(e) => { + warn!( + log, + "Error determining is_aggregator"; + "pubkey" => ?duty.pubkey, + "slot" => proof_slot, + "error" => ?e, + ); + None + } + } + }); // Execute all the futures in parallel, collecting any successful results. let proofs = join_all(futures) @@ -635,7 +650,7 @@ pub async fn fill_in_aggregation_proofs( debug!( log, "Finished computing sync selection proofs"; - "epoch" => epoch, + "slot" => slot, "updated_validators" => num_validators_updated, ); } diff --git a/validator_client/src/lib.rs b/validator_client/src/lib.rs index 4828f43a0d8..52de95a3735 100644 --- a/validator_client/src/lib.rs +++ b/validator_client/src/lib.rs @@ -39,7 +39,7 @@ use account_utils::validator_definitions::ValidatorDefinitions; use attestation_service::{AttestationService, AttestationServiceBuilder}; use block_service::{BlockService, BlockServiceBuilder}; use clap::ArgMatches; -use duties_service::DutiesService; +use duties_service::{sync::SyncDutiesMap, DutiesService}; use environment::RuntimeContext; use eth2::{reqwest::ClientBuilder, types::Graffiti, BeaconNodeHttpClient, StatusCode, Timeouts}; use http_api::ApiSecret; @@ -451,13 +451,14 @@ impl ProductionValidatorClient { let duties_service = Arc::new(DutiesService { attesters: <_>::default(), proposers: <_>::default(), - sync_duties: <_>::default(), + sync_duties: SyncDutiesMap::new(config.distributed), slot_clock: slot_clock.clone(), beacon_nodes: beacon_nodes.clone(), validator_store: validator_store.clone(), spec: context.eth2_config.spec.clone(), context: duties_context, enable_high_validator_count_metrics: config.enable_high_validator_count_metrics, + distributed: config.distributed, }); // Update the metrics server. diff --git a/validator_client/src/sync_committee_service.rs b/validator_client/src/sync_committee_service.rs index 90b62cd3b44..f7abb3855a3 100644 --- a/validator_client/src/sync_committee_service.rs +++ b/validator_client/src/sync_committee_service.rs @@ -161,7 +161,7 @@ impl SyncCommitteeService { let Some(slot_duties) = self .duties_service .sync_duties - .get_duties_for_slot::(slot, &self.duties_service.spec) + .get_duties_for_slot(slot, &self.duties_service.spec) else { debug!(log, "No duties known for slot {}", slot); return Ok(()); @@ -548,7 +548,7 @@ impl SyncCommitteeService { match self .duties_service .sync_duties - .get_duties_for_slot::(duty_slot, spec) + .get_duties_for_slot(duty_slot, spec) { Some(duties) => subscriptions.extend(subscriptions_from_sync_duties( duties.duties, From f17fb291b7ff565e348920d77d605c66d2e113c1 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Thu, 15 Feb 2024 23:24:47 +1100 Subject: [PATCH 15/24] Handle unknown head during attestation publishing (#5010) * Handle unknown head during attestation publishing * Merge remote-tracking branch 'origin/unstable' into queue-http-attestations * Simplify task spawner * Improve logging * Add a test * Improve error logging * Merge remote-tracking branch 'origin/unstable' into queue-http-attestations * Fix beta compiler warnings --- beacon_node/client/src/builder.rs | 4 + beacon_node/http_api/src/lib.rs | 151 ++------- .../http_api/src/publish_attestations.rs | 319 ++++++++++++++++++ beacon_node/http_api/src/task_spawner.rs | 27 +- beacon_node/http_api/src/test_utils.rs | 16 +- .../http_api/tests/interactive_tests.rs | 76 +++++ beacon_node/http_api/tests/tests.rs | 2 + 7 files changed, 455 insertions(+), 140 deletions(-) create mode 100644 beacon_node/http_api/src/publish_attestations.rs diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index 444a277509f..de6e08cc37a 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -505,6 +505,7 @@ where network_senders: None, network_globals: None, beacon_processor_send: None, + beacon_processor_reprocess_send: None, eth1_service: Some(genesis_service.eth1_service.clone()), log: context.log().clone(), sse_logging_components: runtime_context.sse_logging_components.clone(), @@ -747,6 +748,9 @@ where network_globals: self.network_globals.clone(), eth1_service: self.eth1_service.clone(), beacon_processor_send: Some(beacon_processor_channels.beacon_processor_tx.clone()), + beacon_processor_reprocess_send: Some( + beacon_processor_channels.work_reprocessing_tx.clone(), + ), sse_logging_components: runtime_context.sse_logging_components.clone(), log: log.clone(), }); diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index ec3fd494d49..b39450d7354 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -16,6 +16,7 @@ mod database; mod metrics; mod produce_block; mod proposer_duties; +mod publish_attestations; mod publish_blocks; mod standard_block_rewards; mod state_id; @@ -35,7 +36,7 @@ use beacon_chain::{ validator_monitor::timestamp_now, AttestationError as AttnError, BeaconChain, BeaconChainError, BeaconChainTypes, WhenSlotSkipped, }; -use beacon_processor::BeaconProcessorSend; +use beacon_processor::{work_reprocessing_queue::ReprocessQueueMessage, BeaconProcessorSend}; pub use block_id::BlockId; use builder_states::get_next_withdrawals; use bytes::Bytes; @@ -129,6 +130,7 @@ pub struct Context { pub network_senders: Option>, pub network_globals: Option>>, pub beacon_processor_send: Option>, + pub beacon_processor_reprocess_send: Option>, pub eth1_service: Option, pub sse_logging_components: Option, pub log: Logger, @@ -534,6 +536,11 @@ pub fn serve( .filter(|_| config.enable_beacon_processor); let task_spawner_filter = warp::any().map(move || TaskSpawner::new(beacon_processor_send.clone())); + let beacon_processor_reprocess_send = ctx + .beacon_processor_reprocess_send + .clone() + .filter(|_| config.enable_beacon_processor); + let reprocess_send_filter = warp::any().map(move || beacon_processor_reprocess_send.clone()); let duplicate_block_status_code = ctx.config.duplicate_block_status_code; @@ -1756,140 +1763,26 @@ pub fn serve( .and(warp::path::end()) .and(warp_utils::json::json()) .and(network_tx_filter.clone()) + .and(reprocess_send_filter) .and(log_filter.clone()) .then( |task_spawner: TaskSpawner, chain: Arc>, attestations: Vec>, network_tx: UnboundedSender>, - log: Logger| { - task_spawner.blocking_json_task(Priority::P0, move || { - let seen_timestamp = timestamp_now(); - let mut failures = Vec::new(); - let mut num_already_known = 0; - - for (index, attestation) in attestations.as_slice().iter().enumerate() { - let attestation = match chain - .verify_unaggregated_attestation_for_gossip(attestation, None) - { - Ok(attestation) => attestation, - Err(AttnError::PriorAttestationKnown { .. }) => { - num_already_known += 1; - - // Skip to the next attestation since an attestation for this - // validator is already known in this epoch. - // - // There's little value for the network in validating a second - // attestation for another validator since it is either: - // - // 1. A duplicate. - // 2. Slashable. - // 3. Invalid. - // - // We are likely to get duplicates in the case where a VC is using - // fallback BNs. If the first BN actually publishes some/all of a - // batch of attestations but fails to respond in a timely fashion, - // the VC is likely to try publishing the attestations on another - // BN. That second BN may have already seen the attestations from - // the first BN and therefore indicate that the attestations are - // "already seen". An attestation that has already been seen has - // been published on the network so there's no actual error from - // the perspective of the user. - // - // It's better to prevent slashable attestations from ever - // appearing on the network than trying to slash validators, - // especially those validators connected to the local API. - // - // There might be *some* value in determining that this attestation - // is invalid, but since a valid attestation already it exists it - // appears that this validator is capable of producing valid - // attestations and there's no immediate cause for concern. - continue; - } - Err(e) => { - error!(log, - "Failure verifying attestation for gossip"; - "error" => ?e, - "request_index" => index, - "committee_index" => attestation.data.index, - "attestation_slot" => attestation.data.slot, - ); - failures.push(api_types::Failure::new( - index, - format!("Verification: {:?}", e), - )); - // skip to the next attestation so we do not publish this one to gossip - continue; - } - }; - - // Notify the validator monitor. - chain - .validator_monitor - .read() - .register_api_unaggregated_attestation( - seen_timestamp, - attestation.indexed_attestation(), - &chain.slot_clock, - ); - - publish_pubsub_message( - &network_tx, - PubsubMessage::Attestation(Box::new(( - attestation.subnet_id(), - attestation.attestation().clone(), - ))), - )?; - - let committee_index = attestation.attestation().data.index; - let slot = attestation.attestation().data.slot; - - if let Err(e) = chain.apply_attestation_to_fork_choice(&attestation) { - error!(log, - "Failure applying verified attestation to fork choice"; - "error" => ?e, - "request_index" => index, - "committee_index" => committee_index, - "slot" => slot, - ); - failures.push(api_types::Failure::new( - index, - format!("Fork choice: {:?}", e), - )); - }; - - if let Err(e) = chain.add_to_naive_aggregation_pool(&attestation) { - error!(log, - "Failure adding verified attestation to the naive aggregation pool"; - "error" => ?e, - "request_index" => index, - "committee_index" => committee_index, - "slot" => slot, - ); - failures.push(api_types::Failure::new( - index, - format!("Naive aggregation pool: {:?}", e), - )); - } - } - - if num_already_known > 0 { - debug!( - log, - "Some unagg attestations already known"; - "count" => num_already_known - ); - } - - if failures.is_empty() { - Ok(()) - } else { - Err(warp_utils::reject::indexed_bad_request( - "error processing attestations".to_string(), - failures, - )) - } - }) + reprocess_tx: Option>, + log: Logger| async move { + let result = crate::publish_attestations::publish_attestations( + task_spawner, + chain, + attestations, + network_tx, + reprocess_tx, + log, + ) + .await + .map(|()| warp::reply::json(&())); + task_spawner::convert_rejection(result).await }, ); diff --git a/beacon_node/http_api/src/publish_attestations.rs b/beacon_node/http_api/src/publish_attestations.rs new file mode 100644 index 00000000000..ed7f1ed17c9 --- /dev/null +++ b/beacon_node/http_api/src/publish_attestations.rs @@ -0,0 +1,319 @@ +//! Import attestations and publish them to the network. +//! +//! This module gracefully handles attestations to unknown blocks by requeuing them and then +//! efficiently waiting for them to finish reprocessing (using an async yield). +//! +//! The following comments relate to the handling of duplicate attestations (relocated here during +//! refactoring): +//! +//! Skip to the next attestation since an attestation for this +//! validator is already known in this epoch. +//! +//! There's little value for the network in validating a second +//! attestation for another validator since it is either: +//! +//! 1. A duplicate. +//! 2. Slashable. +//! 3. Invalid. +//! +//! We are likely to get duplicates in the case where a VC is using +//! fallback BNs. If the first BN actually publishes some/all of a +//! batch of attestations but fails to respond in a timely fashion, +//! the VC is likely to try publishing the attestations on another +//! BN. That second BN may have already seen the attestations from +//! the first BN and therefore indicate that the attestations are +//! "already seen". An attestation that has already been seen has +//! been published on the network so there's no actual error from +//! the perspective of the user. +//! +//! It's better to prevent slashable attestations from ever +//! appearing on the network than trying to slash validators, +//! especially those validators connected to the local API. +//! +//! There might be *some* value in determining that this attestation +//! is invalid, but since a valid attestation already it exists it +//! appears that this validator is capable of producing valid +//! attestations and there's no immediate cause for concern. +use crate::task_spawner::{Priority, TaskSpawner}; +use beacon_chain::{ + validator_monitor::timestamp_now, AttestationError, BeaconChain, BeaconChainError, + BeaconChainTypes, +}; +use beacon_processor::work_reprocessing_queue::{QueuedUnaggregate, ReprocessQueueMessage}; +use eth2::types::Failure; +use lighthouse_network::PubsubMessage; +use network::NetworkMessage; +use slog::{debug, error, warn, Logger}; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::{ + mpsc::{Sender, UnboundedSender}, + oneshot, +}; +use types::Attestation; + +// Error variants are only used in `Debug` and considered `dead_code` by the compiler. +#[derive(Debug)] +enum Error { + Validation(AttestationError), + Publication, + ForkChoice(#[allow(dead_code)] BeaconChainError), + AggregationPool(#[allow(dead_code)] AttestationError), + ReprocessDisabled, + ReprocessFull, + ReprocessTimeout, +} + +enum PublishAttestationResult { + Success, + AlreadyKnown, + Reprocessing(oneshot::Receiver>), + Failure(Error), +} + +fn verify_and_publish_attestation( + chain: &Arc>, + attestation: &Attestation, + seen_timestamp: Duration, + network_tx: &UnboundedSender>, + log: &Logger, +) -> Result<(), Error> { + let attestation = chain + .verify_unaggregated_attestation_for_gossip(attestation, None) + .map_err(Error::Validation)?; + + // Publish. + network_tx + .send(NetworkMessage::Publish { + messages: vec![PubsubMessage::Attestation(Box::new(( + attestation.subnet_id(), + attestation.attestation().clone(), + )))], + }) + .map_err(|_| Error::Publication)?; + + // Notify the validator monitor. + chain + .validator_monitor + .read() + .register_api_unaggregated_attestation( + seen_timestamp, + attestation.indexed_attestation(), + &chain.slot_clock, + ); + + let fc_result = chain.apply_attestation_to_fork_choice(&attestation); + let naive_aggregation_result = chain.add_to_naive_aggregation_pool(&attestation); + + if let Err(e) = &fc_result { + warn!( + log, + "Attestation invalid for fork choice"; + "err" => ?e, + ); + } + if let Err(e) = &naive_aggregation_result { + warn!( + log, + "Attestation invalid for aggregation"; + "err" => ?e + ); + } + + if let Err(e) = fc_result { + Err(Error::ForkChoice(e)) + } else if let Err(e) = naive_aggregation_result { + Err(Error::AggregationPool(e)) + } else { + Ok(()) + } +} + +pub async fn publish_attestations( + task_spawner: TaskSpawner, + chain: Arc>, + attestations: Vec>, + network_tx: UnboundedSender>, + reprocess_send: Option>, + log: Logger, +) -> Result<(), warp::Rejection> { + // Collect metadata about attestations which we'll use to report failures. We need to + // move the `attestations` vec into the blocking task, so this small overhead is unavoidable. + let attestation_metadata = attestations + .iter() + .map(|att| (att.data.slot, att.data.index)) + .collect::>(); + + // Gossip validate and publish attestations that can be immediately processed. + let seen_timestamp = timestamp_now(); + let inner_log = log.clone(); + let mut prelim_results = task_spawner + .blocking_task(Priority::P0, move || { + Ok(attestations + .into_iter() + .map(|attestation| { + match verify_and_publish_attestation( + &chain, + &attestation, + seen_timestamp, + &network_tx, + &inner_log, + ) { + Ok(()) => PublishAttestationResult::Success, + Err(Error::Validation(AttestationError::UnknownHeadBlock { + beacon_block_root, + })) => { + let Some(reprocess_tx) = &reprocess_send else { + return PublishAttestationResult::Failure(Error::ReprocessDisabled); + }; + // Re-process. + let (tx, rx) = oneshot::channel(); + let reprocess_chain = chain.clone(); + let reprocess_network_tx = network_tx.clone(); + let reprocess_log = inner_log.clone(); + let reprocess_fn = move || { + let result = verify_and_publish_attestation( + &reprocess_chain, + &attestation, + seen_timestamp, + &reprocess_network_tx, + &reprocess_log, + ); + // Ignore failure on the oneshot that reports the result. This + // shouldn't happen unless some catastrophe befalls the waiting + // thread which causes it to drop. + let _ = tx.send(result); + }; + let reprocess_msg = + ReprocessQueueMessage::UnknownBlockUnaggregate(QueuedUnaggregate { + beacon_block_root, + process_fn: Box::new(reprocess_fn), + }); + if reprocess_tx.try_send(reprocess_msg).is_err() { + PublishAttestationResult::Failure(Error::ReprocessFull) + } else { + PublishAttestationResult::Reprocessing(rx) + } + } + Err(Error::Validation(AttestationError::PriorAttestationKnown { + .. + })) => PublishAttestationResult::AlreadyKnown, + Err(e) => PublishAttestationResult::Failure(e), + } + }) + .map(Some) + .collect::>()) + }) + .await?; + + // Asynchronously wait for re-processing of attestations to unknown blocks. This avoids blocking + // any of the beacon processor workers while we wait for reprocessing. + let (reprocess_indices, reprocess_futures): (Vec<_>, Vec<_>) = prelim_results + .iter_mut() + .enumerate() + .filter_map(|(i, opt_result)| { + if let Some(PublishAttestationResult::Reprocessing(..)) = &opt_result { + let PublishAttestationResult::Reprocessing(rx) = opt_result.take()? else { + // Unreachable. + return None; + }; + Some((i, rx)) + } else { + None + } + }) + .unzip(); + let reprocess_results = futures::future::join_all(reprocess_futures).await; + + // Join everything back together and construct a response. + // This part should be quick so we just stay in the Tokio executor's async task. + for (i, reprocess_result) in reprocess_indices.into_iter().zip(reprocess_results) { + let Some(result_entry) = prelim_results.get_mut(i) else { + error!( + log, + "Unreachable case in attestation publishing"; + "case" => "prelim out of bounds", + "request_index" => i, + ); + continue; + }; + *result_entry = Some(match reprocess_result { + Ok(Ok(())) => PublishAttestationResult::Success, + // Attestation failed processing on re-process. + Ok(Err(Error::Validation(AttestationError::PriorAttestationKnown { .. }))) => { + PublishAttestationResult::AlreadyKnown + } + Ok(Err(e)) => PublishAttestationResult::Failure(e), + // Oneshot was dropped, indicating that the attestation either timed out in the + // reprocess queue or was dropped due to some error. + Err(_) => PublishAttestationResult::Failure(Error::ReprocessTimeout), + }); + } + + // Construct the response. + let mut failures = vec![]; + let mut num_already_known = 0; + + for (index, result) in prelim_results.iter().enumerate() { + match result { + Some(PublishAttestationResult::Success) => {} + Some(PublishAttestationResult::AlreadyKnown) => num_already_known += 1, + Some(PublishAttestationResult::Failure(e)) => { + if let Some((slot, committee_index)) = attestation_metadata.get(index) { + error!( + log, + "Failure verifying attestation for gossip"; + "error" => ?e, + "request_index" => index, + "committee_index" => committee_index, + "attestation_slot" => slot, + ); + failures.push(Failure::new(index, format!("{e:?}"))); + } else { + error!( + log, + "Unreachable case in attestation publishing"; + "case" => "out of bounds", + "request_index" => index + ); + failures.push(Failure::new(index, "metadata logic error".into())); + } + } + Some(PublishAttestationResult::Reprocessing(_)) => { + error!( + log, + "Unreachable case in attestation publishing"; + "case" => "reprocessing", + "request_index" => index + ); + failures.push(Failure::new(index, "reprocess logic error".into())); + } + None => { + error!( + log, + "Unreachable case in attestation publishing"; + "case" => "result is None", + "request_index" => index + ); + failures.push(Failure::new(index, "result logic error".into())); + } + } + } + + if num_already_known > 0 { + debug!( + log, + "Some unagg attestations already known"; + "count" => num_already_known + ); + } + + if failures.is_empty() { + Ok(()) + } else { + Err(warp_utils::reject::indexed_bad_request( + "error processing attestations".to_string(), + failures, + )) + } +} diff --git a/beacon_node/http_api/src/task_spawner.rs b/beacon_node/http_api/src/task_spawner.rs index 8768e057dac..cfee5e01ca0 100644 --- a/beacon_node/http_api/src/task_spawner.rs +++ b/beacon_node/http_api/src/task_spawner.rs @@ -60,11 +60,15 @@ impl TaskSpawner { } } - /// Executes a "blocking" (non-async) task which returns a `Response`. - pub async fn blocking_response_task(self, priority: Priority, func: F) -> Response + /// Executes a "blocking" (non-async) task which returns an arbitrary value. + pub async fn blocking_task( + self, + priority: Priority, + func: F, + ) -> Result where F: FnOnce() -> Result + Send + Sync + 'static, - T: Reply + Send + 'static, + T: Send + 'static, { if let Some(beacon_processor_send) = &self.beacon_processor_send { // Create a closure that will execute `func` and send the result to @@ -79,22 +83,31 @@ impl TaskSpawner { }; // Send the function to the beacon processor for execution at some arbitrary time. - let result = send_to_beacon_processor( + send_to_beacon_processor( beacon_processor_send, priority, BlockingOrAsync::Blocking(Box::new(process_fn)), rx, ) .await - .and_then(|x| x); - convert_rejection(result).await + .and_then(|x| x) } else { // There is no beacon processor so spawn a task directly on the // tokio executor. - convert_rejection(warp_utils::task::blocking_response_task(func).await).await + warp_utils::task::blocking_task(func).await } } + /// Executes a "blocking" (non-async) task which returns a `Response`. + pub async fn blocking_response_task(self, priority: Priority, func: F) -> Response + where + F: FnOnce() -> Result + Send + Sync + 'static, + T: Reply + Send + 'static, + { + let result = self.blocking_task(priority, func).await; + convert_rejection(result).await + } + /// Executes a "blocking" (non-async) task which returns a JSON-serializable /// object. pub async fn blocking_json_task(self, priority: Priority, func: F) -> Response diff --git a/beacon_node/http_api/src/test_utils.rs b/beacon_node/http_api/src/test_utils.rs index b87fdf6088f..c1313168bcd 100644 --- a/beacon_node/http_api/src/test_utils.rs +++ b/beacon_node/http_api/src/test_utils.rs @@ -35,6 +35,7 @@ pub const EXTERNAL_ADDR: &str = "/ip4/0.0.0.0/tcp/9000"; /// HTTP API tester that allows interaction with the underlying beacon chain harness. pub struct InteractiveTester { + pub ctx: Arc>>, pub harness: BeaconChainHarness>, pub client: BeaconNodeHttpClient, pub network_rx: NetworkReceivers, @@ -43,10 +44,11 @@ pub struct InteractiveTester { /// The result of calling `create_api_server`. /// /// Glue-type between `tests::ApiTester` and `InteractiveTester`. -pub struct ApiServer> { +pub struct ApiServer> { + pub ctx: Arc>, pub server: SFut, pub listening_socket: SocketAddr, - pub network_rx: NetworkReceivers, + pub network_rx: NetworkReceivers, pub local_enr: Enr, pub external_peer_id: PeerId, } @@ -90,6 +92,7 @@ impl InteractiveTester { let harness = harness_builder.build(); let ApiServer { + ctx, server, listening_socket, network_rx, @@ -114,6 +117,7 @@ impl InteractiveTester { ); Self { + ctx, harness, client, network_rx, @@ -125,7 +129,7 @@ pub async fn create_api_server( chain: Arc>, test_runtime: &TestRuntime, log: Logger, -) -> ApiServer> { +) -> ApiServer> { // Use port 0 to allocate a new unused port. let port = 0; @@ -187,6 +191,7 @@ pub async fn create_api_server( } = BeaconProcessorChannels::new(&beacon_processor_config); let beacon_processor_send = beacon_processor_tx; + let reprocess_send = work_reprocessing_tx.clone(); BeaconProcessor { network_globals: network_globals.clone(), executor: test_runtime.task_executor.clone(), @@ -216,14 +221,17 @@ pub async fn create_api_server( network_senders: Some(network_senders), network_globals: Some(network_globals), beacon_processor_send: Some(beacon_processor_send), + beacon_processor_reprocess_send: Some(reprocess_send), eth1_service: Some(eth1_service), sse_logging_components: None, log, }); - let (listening_socket, server) = crate::serve(ctx, test_runtime.task_executor.exit()).unwrap(); + let (listening_socket, server) = + crate::serve(ctx.clone(), test_runtime.task_executor.exit()).unwrap(); ApiServer { + ctx, server, listening_socket, network_rx: network_receivers, diff --git a/beacon_node/http_api/tests/interactive_tests.rs b/beacon_node/http_api/tests/interactive_tests.rs index 6fb197b41ab..d63d04fcec5 100644 --- a/beacon_node/http_api/tests/interactive_tests.rs +++ b/beacon_node/http_api/tests/interactive_tests.rs @@ -4,6 +4,7 @@ use beacon_chain::{ test_utils::{AttestationStrategy, BlockStrategy, SyncCommitteeStrategy}, ChainConfig, }; +use beacon_processor::work_reprocessing_queue::ReprocessQueueMessage; use eth2::types::ProduceBlockV3Response; use eth2::types::{DepositContractData, StateId}; use execution_layer::{ForkchoiceState, PayloadAttributes}; @@ -840,3 +841,78 @@ pub async fn fork_choice_before_proposal() { // D's parent is B. assert_eq!(block_d.parent_root(), block_root_b.into()); } + +// Test that attestations to unknown blocks are requeued and processed when their block arrives. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn queue_attestations_from_http() { + let validator_count = 128; + let all_validators = (0..validator_count).collect::>(); + + let tester = InteractiveTester::::new(None, validator_count).await; + let harness = &tester.harness; + let client = tester.client.clone(); + + let num_initial = 5; + + // Slot of the block attested to. + let attestation_slot = Slot::new(num_initial) + 1; + + // Make some initial blocks. + harness.advance_slot(); + harness + .extend_chain( + num_initial as usize, + BlockStrategy::OnCanonicalHead, + AttestationStrategy::AllValidators, + ) + .await; + + harness.advance_slot(); + assert_eq!(harness.get_current_slot(), attestation_slot); + + // Make the attested-to block without applying it. + let pre_state = harness.get_current_state(); + let (block, post_state) = harness.make_block(pre_state, attestation_slot).await; + let block_root = block.0.canonical_root(); + + // Make attestations to the block and POST them to the beacon node on a background thread. + let attestations = harness + .make_unaggregated_attestations( + &all_validators, + &post_state, + block.0.state_root(), + block_root.into(), + attestation_slot, + ) + .into_iter() + .flat_map(|attestations| attestations.into_iter().map(|(att, _subnet)| att)) + .collect::>(); + + let attestation_future = tokio::spawn(async move { + client + .post_beacon_pool_attestations(&attestations) + .await + .expect("attestations should be processed successfully") + }); + + // In parallel, apply the block. We need to manually notify the reprocess queue, because the + // `beacon_chain` does not know about the queue and will not update it for us. + let parent_root = block.0.parent_root(); + harness + .process_block(attestation_slot, block_root, block) + .await + .unwrap(); + tester + .ctx + .beacon_processor_reprocess_send + .as_ref() + .unwrap() + .send(ReprocessQueueMessage::BlockImported { + block_root, + parent_root, + }) + .await + .unwrap(); + + attestation_future.await.unwrap(); +} diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 2d946f30921..a7ba2c1ab86 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -248,6 +248,7 @@ impl ApiTester { let log = null_logger().unwrap(); let ApiServer { + ctx: _, server, listening_socket, network_rx, @@ -341,6 +342,7 @@ impl ApiTester { let log = null_logger().unwrap(); let ApiServer { + ctx: _, server, listening_socket, network_rx, From 1711b80779d70122d9f98e3b3115ba67f93e26f2 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Thu, 15 Feb 2024 07:25:02 -0500 Subject: [PATCH 16/24] enable doppelganger tests for deneb (#5137) * enable doppelganger tests for deneb * comment out lcli install skip * Add sanity check * Merge remote-tracking branch 'origin/unstable' into deneb-doppelganger --- .github/workflows/test-suite.yml | 4 +- lcli/src/new_testnet.rs | 74 ++++++++++++++++++++++---------- scripts/local_testnet/geth.sh | 3 +- scripts/tests/genesis.json | 2 +- scripts/tests/vars.env | 9 ++-- 5 files changed, 62 insertions(+), 30 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 77d631a7dab..28b8ec29e33 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -309,7 +309,9 @@ jobs: run: | make - name: Install lcli - if: env.SELF_HOSTED_RUNNERS == 'false' + # TODO: uncomment after the version of lcli in https://github.com/sigp/lighthouse/pull/5137 + # is installed on the runners + # if: env.SELF_HOSTED_RUNNERS == 'false' run: make install-lcli - name: Run the doppelganger protection failure test script run: | diff --git a/lcli/src/new_testnet.rs b/lcli/src/new_testnet.rs index 3a0c7a9f60b..47db1036d98 100644 --- a/lcli/src/new_testnet.rs +++ b/lcli/src/new_testnet.rs @@ -9,7 +9,9 @@ use ethereum_hashing::hash; use ssz::Decode; use ssz::Encode; use state_processing::process_activations; -use state_processing::upgrade::{upgrade_to_altair, upgrade_to_bellatrix}; +use state_processing::upgrade::{ + upgrade_to_altair, upgrade_to_bellatrix, upgrade_to_capella, upgrade_to_deneb, +}; use std::fs::File; use std::io::Read; use std::path::PathBuf; @@ -19,8 +21,8 @@ use types::ExecutionBlockHash; use types::{ test_utils::generate_deterministic_keypairs, Address, BeaconState, ChainSpec, Config, Epoch, Eth1Data, EthSpec, ExecutionPayloadHeader, ExecutionPayloadHeaderCapella, - ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderMerge, ExecutionPayloadHeaderRefMut, - ForkName, Hash256, Keypair, PublicKey, Validator, + ExecutionPayloadHeaderDeneb, ExecutionPayloadHeaderMerge, ForkName, Hash256, Keypair, + PublicKey, Validator, }; pub fn run(testnet_dir_path: PathBuf, matches: &ArgMatches) -> Result<(), String> { @@ -302,26 +304,47 @@ fn initialize_state_with_validators( state.fork_mut().previous_version = spec.bellatrix_fork_version; // Override latest execution payload header. - // See https://github.com/ethereum/consensus-specs/blob/v1.1.0/specs/merge/beacon-chain.md#testing - - // Currently, we only support starting from a bellatrix state - match state - .latest_execution_payload_header_mut() - .map_err(|e| format!("Failed to get execution payload header: {:?}", e))? - { - ExecutionPayloadHeaderRefMut::Merge(header_mut) => { - if let ExecutionPayloadHeader::Merge(eph) = execution_payload_header { - *header_mut = eph; - } else { - return Err("Execution payload header must be a bellatrix header".to_string()); - } - } - ExecutionPayloadHeaderRefMut::Capella(_) => { - return Err("Cannot start genesis from a capella state".to_string()) - } - ExecutionPayloadHeaderRefMut::Deneb(_) => { - return Err("Cannot start genesis from a deneb state".to_string()) - } + // See https://github.com/ethereum/consensus-specs/blob/v1.1.0/specs/bellatrix/beacon-chain.md#testing + if let ExecutionPayloadHeader::Merge(ref header) = execution_payload_header { + *state + .latest_execution_payload_header_merge_mut() + .or(Err("mismatched fork".to_string()))? = header.clone(); + } + } + + if spec + .capella_fork_epoch + .map_or(false, |fork_epoch| fork_epoch == T::genesis_epoch()) + { + upgrade_to_capella(&mut state, spec).unwrap(); + + // Remove intermediate fork from `state.fork`. + state.fork_mut().previous_version = spec.capella_fork_version; + + // Override latest execution payload header. + // See https://github.com/ethereum/consensus-specs/blob/v1.1.0/specs/bellatrix/beacon-chain.md#testing + if let ExecutionPayloadHeader::Capella(ref header) = execution_payload_header { + *state + .latest_execution_payload_header_capella_mut() + .or(Err("mismatched fork".to_string()))? = header.clone(); + } + } + + if spec + .deneb_fork_epoch + .map_or(false, |fork_epoch| fork_epoch == T::genesis_epoch()) + { + upgrade_to_deneb(&mut state, spec).unwrap(); + + // Remove intermediate fork from `state.fork`. + state.fork_mut().previous_version = spec.deneb_fork_version; + + // Override latest execution payload header. + // See https://github.com/ethereum/consensus-specs/blob/v1.1.0/specs/bellatrix/beacon-chain.md#testing + if let ExecutionPayloadHeader::Deneb(ref header) = execution_payload_header { + *state + .latest_execution_payload_header_deneb_mut() + .or(Err("mismatched fork".to_string()))? = header.clone(); } } @@ -331,5 +354,10 @@ fn initialize_state_with_validators( // Set genesis validators root for domain separation and chain versioning *state.genesis_validators_root_mut() = state.update_validators_tree_hash_cache().unwrap(); + // Sanity check for state fork matching config fork. + state + .fork_name(spec) + .map_err(|e| format!("state fork mismatch: {e:?}"))?; + Ok(state) } diff --git a/scripts/local_testnet/geth.sh b/scripts/local_testnet/geth.sh index 5dc4575cf0a..ab1a0ec6ee0 100755 --- a/scripts/local_testnet/geth.sh +++ b/scripts/local_testnet/geth.sh @@ -50,4 +50,5 @@ exec $GETH_BINARY \ --bootnodes $EL_BOOTNODE_ENODE \ --port $network_port \ --http.port $http_port \ - --authrpc.port $auth_port + --authrpc.port $auth_port \ + 2>&1 | tee $data_dir/geth.log diff --git a/scripts/tests/genesis.json b/scripts/tests/genesis.json index 83f45f1a012..bfbc08c81e5 100644 --- a/scripts/tests/genesis.json +++ b/scripts/tests/genesis.json @@ -13,7 +13,7 @@ "londonBlock": 0, "mergeForkBlock": 0, "shanghaiTime": 0, - "shardingForkTime": 0, + "cancunTime": 0, "terminalTotalDifficulty": 0, "terminalTotalDifficultyPassed": true }, diff --git a/scripts/tests/vars.env b/scripts/tests/vars.env index 98ae08f0747..ffe7ac4aecd 100644 --- a/scripts/tests/vars.env +++ b/scripts/tests/vars.env @@ -16,7 +16,7 @@ DEPOSIT_CONTRACT_ADDRESS=4242424242424242424242424242424242424242 GENESIS_FORK_VERSION=0x42424242 # Block hash generated from genesis.json in directory -ETH1_BLOCK_HASH=add7865f8346031c72287e2edc4a4952fd34fc0a8642403e8c1bce67f215c92b +ETH1_BLOCK_HASH=7a5c656343c3a66dcf75415958b500e8873f9dab0cd588e6cf0785b52a06dd34 VALIDATOR_COUNT=80 GENESIS_VALIDATOR_COUNT=80 @@ -41,8 +41,8 @@ CHAIN_ID=4242 # Hard fork configuration ALTAIR_FORK_EPOCH=0 BELLATRIX_FORK_EPOCH=0 -CAPELLA_FORK_EPOCH=1 -DENEB_FORK_EPOCH=18446744073709551615 +CAPELLA_FORK_EPOCH=0 +DENEB_FORK_EPOCH=0 TTD=0 @@ -62,4 +62,5 @@ PROPOSER_SCORE_BOOST=70 BN_ARGS="" # Enable doppelganger detection -VC_ARGS=" --enable-doppelganger-protection " \ No newline at end of file +VC_ARGS=" --enable-doppelganger-protection " + From a264afd19ff1d8d4a9ce1e673386722ccb0e31ec Mon Sep 17 00:00:00 2001 From: ethDreamer <37123614+ethDreamer@users.noreply.github.com> Date: Sun, 18 Feb 2024 23:40:45 +1100 Subject: [PATCH 17/24] Verify Versioned Hashes During Optimistic Sync (#4832) * Convert NewPayloadRequest to use Reference * Refactor for Clarity * Verify Versioned Hashes * Added Tests for Version Hash Verification * Added Moar Tests * Fix Problems Caused By Merge * Update to use Alloy Instead of Reth Crates (#14) * Update beacon_node/execution_layer/src/engine_api/new_payload_request.rs Co-authored-by: realbigsean * Faster Versioned Hash Extraction * Update to rust 1.75 & Pin alloy-consensus --- Cargo.lock | 423 +++++++++++++++++- Dockerfile | 2 +- .../beacon_chain/src/execution_payload.rs | 19 +- beacon_node/execution_layer/Cargo.toml | 2 + beacon_node/execution_layer/src/block_hash.rs | 125 ++---- beacon_node/execution_layer/src/engine_api.rs | 115 +---- .../execution_layer/src/engine_api/http.rs | 6 +- .../src/engine_api/new_payload_request.rs | 332 ++++++++++++++ beacon_node/execution_layer/src/lib.rs | 4 +- .../test_utils/execution_block_generator.rs | 2 +- .../src/test_utils/mock_execution_layer.rs | 2 +- .../execution_layer/src/test_utils/mod.rs | 4 +- .../execution_layer/src/versioned_hashes.rs | 135 ++++++ lcli/Dockerfile | 2 +- lighthouse/Cargo.toml | 2 +- .../src/test_rig.rs | 12 +- 16 files changed, 961 insertions(+), 226 deletions(-) create mode 100644 beacon_node/execution_layer/src/engine_api/new_payload_request.rs create mode 100644 beacon_node/execution_layer/src/versioned_hashes.rs diff --git a/Cargo.lock b/Cargo.lock index 83673d239c0..1c32fed0874 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -179,6 +179,95 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" +[[package]] +name = "alloy-consensus" +version = "0.1.0" +source = "git+https://github.com/alloy-rs/alloy.git?rev=974d488bab5e21e9f17452a39a4bfa56677367b2#974d488bab5e21e9f17452a39a4bfa56677367b2" +dependencies = [ + "alloy-eips", + "alloy-network", + "alloy-primitives", + "alloy-rlp", +] + +[[package]] +name = "alloy-eips" +version = "0.1.0" +source = "git+https://github.com/alloy-rs/alloy.git?rev=974d488bab5e21e9f17452a39a4bfa56677367b2#974d488bab5e21e9f17452a39a4bfa56677367b2" +dependencies = [ + "alloy-primitives", + "alloy-rlp", + "serde", + "thiserror", +] + +[[package]] +name = "alloy-json-rpc" +version = "0.1.0" +source = "git+https://github.com/alloy-rs/alloy.git?rev=974d488bab5e21e9f17452a39a4bfa56677367b2#974d488bab5e21e9f17452a39a4bfa56677367b2" +dependencies = [ + "alloy-primitives", + "serde", + "serde_json", + "thiserror", +] + +[[package]] +name = "alloy-network" +version = "0.1.0" +source = "git+https://github.com/alloy-rs/alloy.git?rev=974d488bab5e21e9f17452a39a4bfa56677367b2#974d488bab5e21e9f17452a39a4bfa56677367b2" +dependencies = [ + "alloy-eips", + "alloy-json-rpc", + "alloy-primitives", + "alloy-rlp", + "serde", +] + +[[package]] +name = "alloy-primitives" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4b6fb2b432ff223d513db7f908937f63c252bee0af9b82bfd25b0a5dd1eb0d8" +dependencies = [ + "alloy-rlp", + "bytes", + "cfg-if", + "const-hex", + "derive_more", + "hex-literal", + "itoa", + "k256 0.13.3", + "keccak-asm", + "proptest", + "rand", + "ruint", + "serde", + "tiny-keccak", +] + +[[package]] +name = "alloy-rlp" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d58d9f5da7b40e9bfff0b7e7816700be4019db97d4b6359fe7f94a9e22e42ac" +dependencies = [ + "alloy-rlp-derive", + "arrayvec", + "bytes", +] + +[[package]] +name = "alloy-rlp-derive" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a047897373be4bbb0224c1afdabca92648dc57a9c9ef6e7b0be3aff7a859c83" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "amcl" version = "0.3.0" @@ -229,6 +318,130 @@ version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +[[package]] +name = "ark-ff" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b3235cc41ee7a12aaaf2c575a2ad7b46713a8a50bda2fc3b003a04845c05dd6" +dependencies = [ + "ark-ff-asm 0.3.0", + "ark-ff-macros 0.3.0", + "ark-serialize 0.3.0", + "ark-std 0.3.0", + "derivative", + "num-bigint", + "num-traits", + "paste", + "rustc_version 0.3.3", + "zeroize", +] + +[[package]] +name = "ark-ff" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec847af850f44ad29048935519032c33da8aa03340876d351dfab5660d2966ba" +dependencies = [ + "ark-ff-asm 0.4.2", + "ark-ff-macros 0.4.2", + "ark-serialize 0.4.2", + "ark-std 0.4.0", + "derivative", + "digest 0.10.7", + "itertools", + "num-bigint", + "num-traits", + "paste", + "rustc_version 0.4.0", + "zeroize", +] + +[[package]] +name = "ark-ff-asm" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db02d390bf6643fb404d3d22d31aee1c4bc4459600aef9113833d17e786c6e44" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ark-ff-asm" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ed4aa4fe255d0bc6d79373f7e31d2ea147bcf486cba1be5ba7ea85abdb92348" +dependencies = [ + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ark-ff-macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db2fd794a08ccb318058009eefdf15bcaaaaf6f8161eb3345f907222bac38b20" +dependencies = [ + "num-bigint", + "num-traits", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ark-ff-macros" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7abe79b0e4288889c4574159ab790824d0033b9fdcb2a112a3182fac2e514565" +dependencies = [ + "num-bigint", + "num-traits", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "ark-serialize" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d6c2b318ee6e10f8c2853e73a83adc0ccb88995aa978d8a3408d492ab2ee671" +dependencies = [ + "ark-std 0.3.0", + "digest 0.9.0", +] + +[[package]] +name = "ark-serialize" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "adb7b85a02b83d2f22f89bd5cac66c9c89474240cb6207cb1efc16d098e822a5" +dependencies = [ + "ark-std 0.4.0", + "digest 0.10.7", + "num-bigint", +] + +[[package]] +name = "ark-std" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1df2c09229cbc5a028b1d70e00fdb2acee28b1055dfb5ca73eea49c5a25c4e7c" +dependencies = [ + "num-traits", + "rand", +] + +[[package]] +name = "ark-std" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94893f1e0c6eeab764ade8dc4c0db24caf4fe7cbbaafc0eba0a9030f447b5185" +dependencies = [ + "num-traits", + "rand", +] + [[package]] name = "arrayref" version = "0.3.7" @@ -485,7 +698,7 @@ checksum = "b6d7b9decdf35d8908a7e3ef02f64c5e9b1695e230154c0e8de3969142d9b94c" dependencies = [ "futures", "pharos", - "rustc_version", + "rustc_version 0.4.0", ] [[package]] @@ -838,6 +1051,21 @@ dependencies = [ "which", ] +[[package]] +name = "bit-set" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1" +dependencies = [ + "bit-vec", +] + +[[package]] +name = "bit-vec" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" + [[package]] name = "bitflags" version = "1.3.2" @@ -1119,7 +1347,7 @@ checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" dependencies = [ "camino", "cargo-platform", - "semver", + "semver 1.0.21", "serde", "serde_json", "thiserror", @@ -1331,6 +1559,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "const-hex" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5104de16b218eddf8e34ffe2f86f74bfa4e61e95a1b89732fccf6325efd0557" +dependencies = [ + "cfg-if", + "cpufeatures", + "hex", + "proptest", + "serde", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -1593,7 +1834,7 @@ dependencies = [ "digest 0.10.7", "fiat-crypto", "platforms 3.3.0", - "rustc_version", + "rustc_version 0.4.0", "subtle", "zeroize", ] @@ -1820,7 +2061,7 @@ dependencies = [ "convert_case", "proc-macro2", "quote", - "rustc_version", + "rustc_version 0.4.0", "syn 1.0.109", ] @@ -2755,6 +2996,8 @@ dependencies = [ name = "execution_layer" version = "0.1.0" dependencies = [ + "alloy-consensus", + "alloy-rlp", "arc-swap", "async-trait", "builder_client", @@ -2848,6 +3091,17 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" +[[package]] +name = "fastrlp" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "139834ddba373bbdd213dffe02c8d110508dcf1726c2be27e8d1f7d7e1856418" +dependencies = [ + "arrayvec", + "auto_impl", + "bytes", +] + [[package]] name = "ff" version = "0.12.1" @@ -2887,7 +3141,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38e2275cc4e4fc009b0669731a1e5ab7ebf11f469eaede2bab9309a5b4d6057f" dependencies = [ "memoffset", - "rustc_version", + "rustc_version 0.4.0", ] [[package]] @@ -3454,6 +3708,15 @@ name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +dependencies = [ + "serde", +] + +[[package]] +name = "hex-literal" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fe2267d4ed49bc07b63801559be28c718ea06c4738b7a03c94df7386d2cde46" [[package]] name = "hex_fmt" @@ -4188,6 +4451,16 @@ dependencies = [ "cpufeatures", ] +[[package]] +name = "keccak-asm" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb8515fff80ed850aea4a1595f2e519c003e2a00a82fe168ebf5269196caf444" +dependencies = [ + "digest 0.10.7", + "sha3-asm", +] + [[package]] name = "keccak-hash" version = "0.10.0" @@ -5618,6 +5891,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" dependencies = [ "autocfg", + "libm", ] [[package]] @@ -5986,6 +6260,17 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "pest" +version = "2.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "219c0dcc30b6a27553f9cc242972b67f75b60eb0db71f0b5462f38b058c41546" +dependencies = [ + "memchr", + "thiserror", + "ucd-trie", +] + [[package]] name = "pharos" version = "0.5.3" @@ -5993,7 +6278,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e9567389417feee6ce15dd6527a8a1ecac205ef62c2932bcf3d9f6fc5b78b414" dependencies = [ "futures", - "rustc_version", + "rustc_version 0.4.0", ] [[package]] @@ -6373,6 +6658,26 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "proptest" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31b476131c3c86cb68032fdc5cb6d5a1045e3e42d96b69fa599fd77701e1f5bf" +dependencies = [ + "bit-set", + "bit-vec", + "bitflags 2.4.2", + "lazy_static", + "num-traits", + "rand", + "rand_chacha", + "rand_xorshift", + "regex-syntax 0.8.2", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "proto_array" version = "0.2.0" @@ -6865,6 +7170,36 @@ dependencies = [ "tokio", ] +[[package]] +name = "ruint" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "608a5726529f2f0ef81b8fde9873c4bb829d6b5b5ca6be4d97345ddf0749c825" +dependencies = [ + "alloy-rlp", + "ark-ff 0.3.0", + "ark-ff 0.4.2", + "bytes", + "fastrlp", + "num-bigint", + "num-traits", + "parity-scale-codec 3.6.9", + "primitive-types 0.12.2", + "proptest", + "rand", + "rlp", + "ruint-macro", + "serde", + "valuable", + "zeroize", +] + +[[package]] +name = "ruint-macro" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e666a5496a0b2186dbcd0ff6106e29e093c15591bde62c20d3842007c6978a09" + [[package]] name = "rusqlite" version = "0.28.0" @@ -6897,13 +7232,22 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e75f6a532d0fd9f7f13144f392b6ad56a32696bfcd9c78f797f16bbb6f072d6" +[[package]] +name = "rustc_version" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" +dependencies = [ + "semver 0.11.0", +] + [[package]] name = "rustc_version" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" dependencies = [ - "semver", + "semver 1.0.21", ] [[package]] @@ -6993,6 +7337,18 @@ version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" +[[package]] +name = "rusty-fork" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb3dcc6e454c328bb824492db107ab7c0ae8fcffe4ad210136ef014458c1bc4f" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "rw-stream-sink" version = "0.4.0" @@ -7159,6 +7515,15 @@ dependencies = [ "libc", ] +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser", +] + [[package]] name = "semver" version = "1.0.21" @@ -7168,6 +7533,15 @@ dependencies = [ "serde", ] +[[package]] +name = "semver-parser" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest", +] + [[package]] name = "send_wrapper" version = "0.6.0" @@ -7367,6 +7741,16 @@ dependencies = [ "keccak", ] +[[package]] +name = "sha3-asm" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bac61da6b35ad76b195eb4771210f947734321a8d81d7738e1580d953bc7a15e" +dependencies = [ + "cc", + "cfg-if", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -7659,7 +8043,7 @@ dependencies = [ "curve25519-dalek", "rand_core", "ring 0.17.7", - "rustc_version", + "rustc_version 0.4.0", "sha2 0.10.8", "subtle", ] @@ -8621,6 +9005,12 @@ dependencies = [ "tree_hash_derive", ] +[[package]] +name = "ucd-trie" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed646292ffc8188ef8ea4d1e0e0150fb15a5c2e12ad9b8fc191ae7a8a7f3c4b9" + [[package]] name = "uint" version = "0.9.5" @@ -8634,6 +9024,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "unescape" version = "0.1.0" @@ -8911,6 +9307,15 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" +[[package]] +name = "wait-timeout" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f200f5b12eb75f8c1ed65abd4b2db8a6e1b138a20de009dacee265a2498f3f6" +dependencies = [ + "libc", +] + [[package]] name = "waker-fn" version = "1.1.1" @@ -9496,7 +9901,7 @@ dependencies = [ "js-sys", "log", "pharos", - "rustc_version", + "rustc_version 0.4.0", "send_wrapper", "thiserror", "wasm-bindgen", diff --git a/Dockerfile b/Dockerfile index a8dadf2ad57..901c1b83d63 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM rust:1.73.0-bullseye AS builder +FROM rust:1.75.0-bullseye AS builder RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev COPY . lighthouse ARG FEATURES diff --git a/beacon_node/beacon_chain/src/execution_payload.rs b/beacon_node/beacon_chain/src/execution_payload.rs index e25976c2a57..fd790c88429 100644 --- a/beacon_node/beacon_chain/src/execution_payload.rs +++ b/beacon_node/beacon_chain/src/execution_payload.rs @@ -81,14 +81,10 @@ impl PayloadNotifier { match notify_execution_layer { NotifyExecutionLayer::No if chain.config.optimistic_finalized_sync => { - // Verify the block hash here in Lighthouse and immediately mark the block as - // optimistically imported. This saves a lot of roundtrips to the EL. - let execution_layer = chain - .execution_layer - .as_ref() - .ok_or(ExecutionPayloadError::NoExecutionConnection)?; - - if let Err(e) = execution_layer.verify_payload_block_hash(block_message) { + // Create a NewPayloadRequest (no clones required) and check optimistic sync verifications + let new_payload_request: NewPayloadRequest = + block_message.try_into()?; + if let Err(e) = new_payload_request.perform_optimistic_sync_verifications() { warn!( chain.log, "Falling back to slow block hash verification"; @@ -143,11 +139,8 @@ async fn notify_new_payload<'a, T: BeaconChainTypes>( .as_ref() .ok_or(ExecutionPayloadError::NoExecutionConnection)?; - let new_payload_request: NewPayloadRequest = block.try_into()?; - let execution_block_hash = new_payload_request.block_hash(); - let new_payload_response = execution_layer - .notify_new_payload(new_payload_request) - .await; + let execution_block_hash = block.execution_payload()?.block_hash(); + let new_payload_response = execution_layer.notify_new_payload(block.try_into()?).await; match new_payload_response { Ok(status) => match status { diff --git a/beacon_node/execution_layer/Cargo.toml b/beacon_node/execution_layer/Cargo.toml index 7f652689806..7fee3721d8f 100644 --- a/beacon_node/execution_layer/Cargo.toml +++ b/beacon_node/execution_layer/Cargo.toml @@ -51,3 +51,5 @@ hash-db = "0.15.2" pretty_reqwest_error = { workspace = true } arc-swap = "1.6.0" eth2_network_config = { workspace = true } +alloy-rlp = "0.3" +alloy-consensus = { git = "https://github.com/alloy-rs/alloy.git", rev = "974d488bab5e21e9f17452a39a4bfa56677367b2" } diff --git a/beacon_node/execution_layer/src/block_hash.rs b/beacon_node/execution_layer/src/block_hash.rs index 5ba61beafca..074ef8b0c14 100644 --- a/beacon_node/execution_layer/src/block_hash.rs +++ b/beacon_node/execution_layer/src/block_hash.rs @@ -1,92 +1,61 @@ use crate::{ json_structures::JsonWithdrawal, keccak::{keccak256, KeccakHasher}, - metrics, Error, ExecutionLayer, }; use ethers_core::utils::rlp::RlpStream; use keccak_hash::KECCAK_EMPTY_LIST_RLP; use triehash::ordered_trie_root; use types::{ - map_execution_block_header_fields_base, Address, BeaconBlockRef, EthSpec, ExecutionBlockHash, + map_execution_block_header_fields_base, Address, EthSpec, ExecutionBlockHash, ExecutionBlockHeader, ExecutionPayloadRef, Hash256, Hash64, Uint256, }; -impl ExecutionLayer { - /// Calculate the block hash of an execution block. - /// - /// Return `(block_hash, transactions_root)`, where `transactions_root` is the root of the RLP - /// transactions. - pub fn calculate_execution_block_hash( - payload: ExecutionPayloadRef, - parent_beacon_block_root: Hash256, - ) -> (ExecutionBlockHash, Hash256) { - // Calculate the transactions root. - // We're currently using a deprecated Parity library for this. We should move to a - // better alternative when one appears, possibly following Reth. - let rlp_transactions_root = ordered_trie_root::( - payload.transactions().iter().map(|txn_bytes| &**txn_bytes), - ); - - // Calculate withdrawals root (post-Capella). - let rlp_withdrawals_root = if let Ok(withdrawals) = payload.withdrawals() { - Some(ordered_trie_root::( - withdrawals.iter().map(|withdrawal| { - rlp_encode_withdrawal(&JsonWithdrawal::from(withdrawal.clone())) - }), - )) - } else { - None - }; - - let rlp_blob_gas_used = payload.blob_gas_used().ok(); - let rlp_excess_blob_gas = payload.excess_blob_gas().ok(); - - // Calculate parent beacon block root (post-Deneb). - let rlp_parent_beacon_block_root = rlp_excess_blob_gas - .as_ref() - .map(|_| parent_beacon_block_root); - - // Construct the block header. - let exec_block_header = ExecutionBlockHeader::from_payload( - payload, - KECCAK_EMPTY_LIST_RLP.as_fixed_bytes().into(), - rlp_transactions_root, - rlp_withdrawals_root, - rlp_blob_gas_used, - rlp_excess_blob_gas, - rlp_parent_beacon_block_root, - ); - - // Hash the RLP encoding of the block header. - let rlp_block_header = rlp_encode_block_header(&exec_block_header); - ( - ExecutionBlockHash::from_root(keccak256(&rlp_block_header)), - rlp_transactions_root, - ) - } - - /// Verify `payload.block_hash` locally within Lighthouse. - /// - /// No remote calls to the execution client will be made, so this is quite a cheap check. - pub fn verify_payload_block_hash(&self, block: BeaconBlockRef) -> Result<(), Error> { - let payload = block.execution_payload()?.execution_payload_ref(); - let parent_beacon_block_root = block.parent_root(); - - let _timer = metrics::start_timer(&metrics::EXECUTION_LAYER_VERIFY_BLOCK_HASH); - - let (header_hash, rlp_transactions_root) = - Self::calculate_execution_block_hash(payload, parent_beacon_block_root); - - if header_hash != payload.block_hash() { - return Err(Error::BlockHashMismatch { - computed: header_hash, - payload: payload.block_hash(), - transactions_root: rlp_transactions_root, - }); - } - - Ok(()) - } +/// Calculate the block hash of an execution block. +/// +/// Return `(block_hash, transactions_root)`, where `transactions_root` is the root of the RLP +/// transactions. +pub fn calculate_execution_block_hash( + payload: ExecutionPayloadRef, + parent_beacon_block_root: Option, +) -> (ExecutionBlockHash, Hash256) { + // Calculate the transactions root. + // We're currently using a deprecated Parity library for this. We should move to a + // better alternative when one appears, possibly following Reth. + let rlp_transactions_root = ordered_trie_root::( + payload.transactions().iter().map(|txn_bytes| &**txn_bytes), + ); + + // Calculate withdrawals root (post-Capella). + let rlp_withdrawals_root = if let Ok(withdrawals) = payload.withdrawals() { + Some(ordered_trie_root::( + withdrawals + .iter() + .map(|withdrawal| rlp_encode_withdrawal(&JsonWithdrawal::from(withdrawal.clone()))), + )) + } else { + None + }; + + let rlp_blob_gas_used = payload.blob_gas_used().ok(); + let rlp_excess_blob_gas = payload.excess_blob_gas().ok(); + + // Construct the block header. + let exec_block_header = ExecutionBlockHeader::from_payload( + payload, + KECCAK_EMPTY_LIST_RLP.as_fixed_bytes().into(), + rlp_transactions_root, + rlp_withdrawals_root, + rlp_blob_gas_used, + rlp_excess_blob_gas, + parent_beacon_block_root, + ); + + // Hash the RLP encoding of the block header. + let rlp_block_header = rlp_encode_block_header(&exec_block_header); + ( + ExecutionBlockHash::from_root(keccak256(&rlp_block_header)), + rlp_transactions_root, + ) } /// RLP encode a withdrawal. diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index 19b9a58eb66..e20009e2858 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -17,7 +17,6 @@ pub use json_structures::{JsonWithdrawal, TransitionConfigurationV1}; use pretty_reqwest_error::PrettyReqwestError; use reqwest::StatusCode; use serde::{Deserialize, Serialize}; -use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; use std::convert::TryFrom; use strum::IntoStaticStr; use superstruct::superstruct; @@ -26,14 +25,16 @@ pub use types::{ ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, Uint256, VariableList, Withdrawal, Withdrawals, }; -use types::{ - BeaconStateError, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, - KzgProofs, VersionedHash, -}; +use types::{ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, KzgProofs}; pub mod auth; pub mod http; pub mod json_structures; +mod new_payload_request; + +pub use new_payload_request::{ + NewPayloadRequest, NewPayloadRequestCapella, NewPayloadRequestDeneb, NewPayloadRequestMerge, +}; pub const LATEST_TAG: &str = "latest"; @@ -571,110 +572,6 @@ impl ExecutionPayloadBodyV1 { } } -#[superstruct( - variants(Merge, Capella, Deneb), - variant_attributes(derive(Clone, Debug, PartialEq),), - map_into(ExecutionPayload), - map_ref_into(ExecutionPayloadRef), - cast_error( - ty = "BeaconStateError", - expr = "BeaconStateError::IncorrectStateVariant" - ), - partial_getter_error( - ty = "BeaconStateError", - expr = "BeaconStateError::IncorrectStateVariant" - ) -)] -#[derive(Clone, Debug, PartialEq)] -pub struct NewPayloadRequest { - #[superstruct(only(Merge), partial_getter(rename = "execution_payload_merge"))] - pub execution_payload: ExecutionPayloadMerge, - #[superstruct(only(Capella), partial_getter(rename = "execution_payload_capella"))] - pub execution_payload: ExecutionPayloadCapella, - #[superstruct(only(Deneb), partial_getter(rename = "execution_payload_deneb"))] - pub execution_payload: ExecutionPayloadDeneb, - #[superstruct(only(Deneb))] - pub versioned_hashes: Vec, - #[superstruct(only(Deneb))] - pub parent_beacon_block_root: Hash256, -} - -impl NewPayloadRequest { - pub fn parent_hash(&self) -> ExecutionBlockHash { - match self { - Self::Merge(payload) => payload.execution_payload.parent_hash, - Self::Capella(payload) => payload.execution_payload.parent_hash, - Self::Deneb(payload) => payload.execution_payload.parent_hash, - } - } - - pub fn block_hash(&self) -> ExecutionBlockHash { - match self { - Self::Merge(payload) => payload.execution_payload.block_hash, - Self::Capella(payload) => payload.execution_payload.block_hash, - Self::Deneb(payload) => payload.execution_payload.block_hash, - } - } - - pub fn block_number(&self) -> u64 { - match self { - Self::Merge(payload) => payload.execution_payload.block_number, - Self::Capella(payload) => payload.execution_payload.block_number, - Self::Deneb(payload) => payload.execution_payload.block_number, - } - } - - pub fn into_execution_payload(self) -> ExecutionPayload { - map_new_payload_request_into_execution_payload!(self, |request, cons| { - cons(request.execution_payload) - }) - } -} - -impl<'a, E: EthSpec> TryFrom> for NewPayloadRequest { - type Error = BeaconStateError; - - fn try_from(block: BeaconBlockRef<'a, E>) -> Result { - match block { - BeaconBlockRef::Base(_) | BeaconBlockRef::Altair(_) => { - Err(Self::Error::IncorrectStateVariant) - } - BeaconBlockRef::Merge(block_ref) => Ok(Self::Merge(NewPayloadRequestMerge { - execution_payload: block_ref.body.execution_payload.execution_payload.clone(), - })), - BeaconBlockRef::Capella(block_ref) => Ok(Self::Capella(NewPayloadRequestCapella { - execution_payload: block_ref.body.execution_payload.execution_payload.clone(), - })), - BeaconBlockRef::Deneb(block_ref) => Ok(Self::Deneb(NewPayloadRequestDeneb { - execution_payload: block_ref.body.execution_payload.execution_payload.clone(), - versioned_hashes: block_ref - .body - .blob_kzg_commitments - .iter() - .map(kzg_commitment_to_versioned_hash) - .collect(), - parent_beacon_block_root: block_ref.parent_root, - })), - } - } -} - -impl TryFrom> for NewPayloadRequest { - type Error = BeaconStateError; - - fn try_from(payload: ExecutionPayload) -> Result { - match payload { - ExecutionPayload::Merge(payload) => Ok(Self::Merge(NewPayloadRequestMerge { - execution_payload: payload, - })), - ExecutionPayload::Capella(payload) => Ok(Self::Capella(NewPayloadRequestCapella { - execution_payload: payload, - })), - ExecutionPayload::Deneb(_) => Err(Self::Error::IncorrectStateVariant), - } - } -} - #[derive(Clone, Copy, Debug)] pub struct EngineCapabilities { pub new_payload_v1: bool, diff --git a/beacon_node/execution_layer/src/engine_api/http.rs b/beacon_node/execution_layer/src/engine_api/http.rs index ac7dfa57e92..df0f79c61e2 100644 --- a/beacon_node/execution_layer/src/engine_api/http.rs +++ b/beacon_node/execution_layer/src/engine_api/http.rs @@ -803,10 +803,10 @@ impl HttpJsonRpc { pub async fn new_payload_v3( &self, - new_payload_request_deneb: NewPayloadRequestDeneb, + new_payload_request_deneb: NewPayloadRequestDeneb<'_, T>, ) -> Result { let params = json!([ - JsonExecutionPayload::V3(new_payload_request_deneb.execution_payload.into()), + JsonExecutionPayload::V3(new_payload_request_deneb.execution_payload.clone().into()), new_payload_request_deneb.versioned_hashes, new_payload_request_deneb.parent_beacon_block_root, ]); @@ -1079,7 +1079,7 @@ impl HttpJsonRpc { // new_payload that the execution engine supports pub async fn new_payload( &self, - new_payload_request: NewPayloadRequest, + new_payload_request: NewPayloadRequest<'_, T>, ) -> Result { let engine_capabilities = self.get_engine_capabilities(None).await?; match new_payload_request { diff --git a/beacon_node/execution_layer/src/engine_api/new_payload_request.rs b/beacon_node/execution_layer/src/engine_api/new_payload_request.rs new file mode 100644 index 00000000000..b1385399e89 --- /dev/null +++ b/beacon_node/execution_layer/src/engine_api/new_payload_request.rs @@ -0,0 +1,332 @@ +use crate::{block_hash::calculate_execution_block_hash, metrics, Error}; + +use crate::versioned_hashes::verify_versioned_hashes; +use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; +use superstruct::superstruct; +use types::{ + BeaconBlockRef, BeaconStateError, EthSpec, ExecutionBlockHash, ExecutionPayload, + ExecutionPayloadRef, Hash256, VersionedHash, +}; +use types::{ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge}; + +#[superstruct( + variants(Merge, Capella, Deneb), + variant_attributes(derive(Clone, Debug, PartialEq),), + map_into(ExecutionPayload), + map_ref_into(ExecutionPayloadRef), + cast_error( + ty = "BeaconStateError", + expr = "BeaconStateError::IncorrectStateVariant" + ), + partial_getter_error( + ty = "BeaconStateError", + expr = "BeaconStateError::IncorrectStateVariant" + ) +)] +#[derive(Clone, Debug, PartialEq)] +pub struct NewPayloadRequest<'block, E: EthSpec> { + #[superstruct(only(Merge), partial_getter(rename = "execution_payload_merge"))] + pub execution_payload: &'block ExecutionPayloadMerge, + #[superstruct(only(Capella), partial_getter(rename = "execution_payload_capella"))] + pub execution_payload: &'block ExecutionPayloadCapella, + #[superstruct(only(Deneb), partial_getter(rename = "execution_payload_deneb"))] + pub execution_payload: &'block ExecutionPayloadDeneb, + #[superstruct(only(Deneb))] + pub versioned_hashes: Vec, + #[superstruct(only(Deneb))] + pub parent_beacon_block_root: Hash256, +} + +impl<'block, E: EthSpec> NewPayloadRequest<'block, E> { + pub fn parent_hash(&self) -> ExecutionBlockHash { + match self { + Self::Merge(payload) => payload.execution_payload.parent_hash, + Self::Capella(payload) => payload.execution_payload.parent_hash, + Self::Deneb(payload) => payload.execution_payload.parent_hash, + } + } + + pub fn block_hash(&self) -> ExecutionBlockHash { + match self { + Self::Merge(payload) => payload.execution_payload.block_hash, + Self::Capella(payload) => payload.execution_payload.block_hash, + Self::Deneb(payload) => payload.execution_payload.block_hash, + } + } + + pub fn block_number(&self) -> u64 { + match self { + Self::Merge(payload) => payload.execution_payload.block_number, + Self::Capella(payload) => payload.execution_payload.block_number, + Self::Deneb(payload) => payload.execution_payload.block_number, + } + } + + pub fn execution_payload_ref(&self) -> ExecutionPayloadRef<'block, E> { + match self { + Self::Merge(request) => ExecutionPayloadRef::Merge(request.execution_payload), + Self::Capella(request) => ExecutionPayloadRef::Capella(request.execution_payload), + Self::Deneb(request) => ExecutionPayloadRef::Deneb(request.execution_payload), + } + } + + pub fn into_execution_payload(self) -> ExecutionPayload { + match self { + Self::Merge(request) => ExecutionPayload::Merge(request.execution_payload.clone()), + Self::Capella(request) => ExecutionPayload::Capella(request.execution_payload.clone()), + Self::Deneb(request) => ExecutionPayload::Deneb(request.execution_payload.clone()), + } + } + + /// Performs the required verifications of the payload when the chain is optimistically syncing. + /// + /// ## Specification + /// + /// Performs the verifications in the `verify_and_notify_new_payload` function: + /// + /// https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/beacon-chain.md#modified-verify_and_notify_new_payload + pub fn perform_optimistic_sync_verifications(&self) -> Result<(), Error> { + self.verfiy_payload_block_hash()?; + self.verify_versioned_hashes()?; + + Ok(()) + } + + /// Verify the block hash is consistent locally within Lighthouse. + /// + /// ## Specification + /// + /// Equivalent to `is_valid_block_hash` in the spec: + /// https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/beacon-chain.md#is_valid_block_hash + pub fn verfiy_payload_block_hash(&self) -> Result<(), Error> { + let payload = self.execution_payload_ref(); + let parent_beacon_block_root = self.parent_beacon_block_root().ok().cloned(); + + let _timer = metrics::start_timer(&metrics::EXECUTION_LAYER_VERIFY_BLOCK_HASH); + + let (header_hash, rlp_transactions_root) = + calculate_execution_block_hash(payload, parent_beacon_block_root); + + if header_hash != self.block_hash() { + return Err(Error::BlockHashMismatch { + computed: header_hash, + payload: payload.block_hash(), + transactions_root: rlp_transactions_root, + }); + } + + Ok(()) + } + + /// Verify the versioned hashes computed by the blob transactions match the versioned hashes computed from the commitments. + /// + /// ## Specification + /// + /// Equivalent to `is_valid_versioned_hashes` in the spec: + /// https://github.com/ethereum/consensus-specs/blob/v1.4.0-beta.2/specs/deneb/beacon-chain.md#is_valid_versioned_hashes + pub fn verify_versioned_hashes(&self) -> Result<(), Error> { + if let Ok(versioned_hashes) = self.versioned_hashes() { + verify_versioned_hashes(self.execution_payload_ref(), versioned_hashes) + .map_err(Error::VerifyingVersionedHashes)?; + } + Ok(()) + } +} + +impl<'a, E: EthSpec> TryFrom> for NewPayloadRequest<'a, E> { + type Error = BeaconStateError; + + fn try_from(block: BeaconBlockRef<'a, E>) -> Result { + match block { + BeaconBlockRef::Base(_) | BeaconBlockRef::Altair(_) => { + Err(Self::Error::IncorrectStateVariant) + } + BeaconBlockRef::Merge(block_ref) => Ok(Self::Merge(NewPayloadRequestMerge { + execution_payload: &block_ref.body.execution_payload.execution_payload, + })), + BeaconBlockRef::Capella(block_ref) => Ok(Self::Capella(NewPayloadRequestCapella { + execution_payload: &block_ref.body.execution_payload.execution_payload, + })), + BeaconBlockRef::Deneb(block_ref) => Ok(Self::Deneb(NewPayloadRequestDeneb { + execution_payload: &block_ref.body.execution_payload.execution_payload, + versioned_hashes: block_ref + .body + .blob_kzg_commitments + .iter() + .map(kzg_commitment_to_versioned_hash) + .collect(), + parent_beacon_block_root: block_ref.parent_root, + })), + } + } +} + +impl<'a, E: EthSpec> TryFrom> for NewPayloadRequest<'a, E> { + type Error = BeaconStateError; + + fn try_from(payload: ExecutionPayloadRef<'a, E>) -> Result { + match payload { + ExecutionPayloadRef::Merge(payload) => Ok(Self::Merge(NewPayloadRequestMerge { + execution_payload: payload, + })), + ExecutionPayloadRef::Capella(payload) => Ok(Self::Capella(NewPayloadRequestCapella { + execution_payload: payload, + })), + ExecutionPayloadRef::Deneb(_) => Err(Self::Error::IncorrectStateVariant), + } + } +} + +#[cfg(test)] +mod test { + use crate::versioned_hashes::Error as VersionedHashError; + use crate::{Error, NewPayloadRequest}; + use state_processing::per_block_processing::deneb::kzg_commitment_to_versioned_hash; + use types::{BeaconBlock, ExecPayload, ExecutionBlockHash, Hash256, MainnetEthSpec}; + + #[test] + fn test_optimistic_sync_verifications_valid_block() { + let beacon_block = get_valid_beacon_block(); + let new_payload_request = NewPayloadRequest::try_from(beacon_block.to_ref()) + .expect("should create new payload request"); + + assert!( + new_payload_request + .perform_optimistic_sync_verifications() + .is_ok(), + "validations should pass" + ); + } + + #[test] + fn test_optimistic_sync_verifications_bad_block_hash() { + let mut beacon_block = get_valid_beacon_block(); + let correct_block_hash = beacon_block + .body() + .execution_payload() + .expect("should get payload") + .block_hash(); + let invalid_block_hash = ExecutionBlockHash(Hash256::repeat_byte(0x42)); + + // now mutate the block hash + beacon_block + .body_mut() + .execution_payload_deneb_mut() + .expect("should get payload") + .execution_payload + .block_hash = invalid_block_hash; + + let new_payload_request = NewPayloadRequest::try_from(beacon_block.to_ref()) + .expect("should create new payload request"); + let verification_result = new_payload_request.perform_optimistic_sync_verifications(); + println!("verification_result: {:?}", verification_result); + let got_expected_result = match verification_result { + Err(Error::BlockHashMismatch { + computed, payload, .. + }) => computed == correct_block_hash && payload == invalid_block_hash, + _ => false, + }; + assert!(got_expected_result, "should return expected error"); + } + + #[test] + fn test_optimistic_sync_verifications_bad_versioned_hashes() { + let mut beacon_block = get_valid_beacon_block(); + + let mut commitments: Vec<_> = beacon_block + .body() + .blob_kzg_commitments() + .expect("should get commitments") + .clone() + .into(); + + let correct_versioned_hash = kzg_commitment_to_versioned_hash( + commitments.last().expect("should get last commitment"), + ); + + // mutate the last commitment + commitments + .last_mut() + .expect("should get last commitment") + .0[0] = 0x42; + + // calculate versioned hash from mutated commitment + let bad_versioned_hash = kzg_commitment_to_versioned_hash( + commitments.last().expect("should get last commitment"), + ); + + *beacon_block + .body_mut() + .blob_kzg_commitments_mut() + .expect("should get commitments") = commitments.into(); + + let new_payload_request = NewPayloadRequest::try_from(beacon_block.to_ref()) + .expect("should create new payload request"); + let verification_result = new_payload_request.perform_optimistic_sync_verifications(); + println!("verification_result: {:?}", verification_result); + + let got_expected_result = match verification_result { + Err(Error::VerifyingVersionedHashes(VersionedHashError::VersionHashMismatch { + expected, + found, + })) => expected == bad_versioned_hash && found == correct_versioned_hash, + _ => false, + }; + assert!(got_expected_result, "should return expected error"); + } + + fn get_valid_beacon_block() -> BeaconBlock { + BeaconBlock::Deneb(serde_json::from_str(r#"{ + "slot": "88160", + "proposer_index": "583", + "parent_root": "0x60770cd86a497ca3aa2e91f1687aa3ebafac87af52c30a920b5f40bd9e930eb6", + "state_root": "0x4a0e0abbcbcf576f2cb7387c4289ab13b8a128e32127642f056143d6164941a6", + "body": { + "randao_reveal": "0xb5253d5739496abc4f67c7c92e39e46cca452c2fdfc5275e3e0426a012aa62df82f47f7dece348e28db4bb212f0e793d187120bbd47b8031ed79344116eb4128f0ce0b05ba18cd615bb13966c1bd7d89e23cc769c8e4d8e4a63755f623ac3bed", + "eth1_data": { + "deposit_root": "0xe4785ac914d8673797f886e3151ce2647f81ae070c7ddb6845e65fd1c47d1222", + "deposit_count": "1181", + "block_hash": "0x010671bdfbfce6b0071984a06a7ded6deef13b4f8fdbae402c606a7a0c8780d1" + }, + "graffiti": "0x6c6f6465737461722f6765746800000000000000000000000000000000000000", + "proposer_slashings": [], + "attester_slashings": [], + "attestations": [], + "deposits": [], + "voluntary_exits": [], + "sync_aggregate": { + "sync_committee_bits": "0xfebffffffebfff7fff7f7fffbbefffff6affffffffbfffffefffebfffdbf77fff7fd77ffffefffdff7ffffeffffffe7e5ffffffdefffff7ffbffff7fffffffff", + "sync_committee_signature": "0x91939b5baf2a6f52d405b6dd396f5346ec435eca7d25912c91cc6a2f7030d870d68bebe4f2b21872a06929ff4cf3e5e9191053cb43eb24ebe34b9a75fb88a3acd06baf329c87f68bd664b49891260c698d7bca0f5365870b5b2b3a76f582156c" + }, + "execution_payload": { + "parent_hash": "0xa6f3ed782a992f79ad38da2af91b3e8923c71b801c50bc9033bb35a2e1da885f", + "fee_recipient": "0xf97e180c050e5ab072211ad2c213eb5aee4df134", + "state_root": "0x3bfd1a7f309ed35048c349a8daf01815bdc09a6d5df86ea77d1056f248ba2017", + "receipts_root": "0xcb5b8ffea57cd0fa87194d49bc8bb7fad08c93c9934b886489503c328d15fd36", + "logs_bloom": "0x002000000000000000000000800000000000000000001040000000000000000000000001000000000000000000000000000000000000100000000020000c0800000000000000008000000008000000200000800000000000000000000000000000000000000000008000000000008000000000000000000002000010000000000000000000000000000000000000000000000000000000080000004000000000800000000000000000000100000000000000000000000000000000000800000000000102000000000000000000000000000000080000001000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "prev_randao": "0xb2693020177d99ffbd4c267023be172d759e7306ff51b0e7d677d3148fbd7f1d", + "block_number": "74807", + "gas_limit": "30000000", + "gas_used": "128393", + "timestamp": "1697039520", + "extra_data": "0xd883010d03846765746888676f312e32312e31856c696e7578", + "base_fee_per_gas": "7", + "block_hash": "0xc64f3a43c64aeb98518a237f6279fa03095b9f95ca673c860ad7f16fb9340062", + "transactions": [ + "0x02f9017a8501a1f0ff4382317585012a05f2008512a05f2000830249f094c1b0bc605e2c808aa0867bfc98e51a1fe3e9867f80b901040cc7326300000000000000000000000000000000000000000000000000000000000000a0000000000000000000000000000000000000000000000000036e534e16b8920d000000000000000000000000fb3e9c7cb92443931ee6b5b9728598d4eb9618c1000000000000000000000000fc7360b3b28cf4204268a8354dbec60720d155d2000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000009a054a063f0fe7b9c68de8df91aaa5e96c15ab540000000000000000000000000c8d41b8fcc066cdabaf074d78e5153e8ce018a9c080a07dd9be0d014ffcd5b6883d0917c66b74ba51f0d976c8fc5674af192af6fa9450a02dad2c660974c125f5f22b1e6e8862a292e08cc2b4cafda35af650ee62868a43", + "0x03f8db8501a1f0ff430d84773594008504a817c8008252089454e594b6de0aa4b0188cd1549dd7ba715a455d078080c08504a817c800f863a001253ce00f525e3495cffa0b865eadb90a4c5ee812185cc796af74b6ec0a5dd7a0010720372a4d7dcab84413ed0cfc164fb91fb6ef1562ec2f7a82e912a1d9e129a0015a73e97950397896ed2c47dcab7c0360220bcfb413a8f210a7b6e6264e698880a04402cb0f13c17ef41dca106b1e1520c7aadcbe62984d81171e29914f587d67c1a02db62a8edb581917958e4a3884e7eececbaec114c5ee496e238033e896f997ac" + ], + "withdrawals": [], + "blob_gas_used": "393216", + "excess_blob_gas": "58720256" + }, + "bls_to_execution_changes": [], + "blob_kzg_commitments": [ + "0xa7accb7a25224a8c2e0cee9cd569fc1798665bfbfe780e08945fa9098ec61da4061f5b04e750a88d3340a801850a54fa", + "0xac7b47f99836510ae9076dc5f5da1f370679dea1d47073307a14cbb125cdc7822ae619637135777cb40e13d897fd00a7", + "0x997794110b9655833a88ad5a4ec40a3dc7964877bfbeb04ca1abe1d51bdc43e20e4c5757028896d298d7da954a6f14a1" + ] + } + }"#).expect("should decode")) + } +} diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 868d8194466..664ceabb6cd 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -61,6 +61,7 @@ mod metrics; pub mod payload_cache; mod payload_status; pub mod test_utils; +mod versioned_hashes; /// Indicates the default jwt authenticated execution endpoint. pub const DEFAULT_EXECUTION_ENDPOINT: &str = "http://localhost:8551/"; @@ -141,6 +142,7 @@ pub enum Error { InvalidBlobConversion(String), BeaconStateError(BeaconStateError), PayloadTypeMismatch, + VerifyingVersionedHashes(versioned_hashes::Error), } impl From for Error { @@ -1321,7 +1323,7 @@ impl ExecutionLayer { /// Maps to the `engine_newPayload` JSON-RPC call. pub async fn notify_new_payload( &self, - new_payload_request: NewPayloadRequest, + new_payload_request: NewPayloadRequest<'_, T>, ) -> Result { let _timer = metrics::start_timer_vec( &metrics::EXECUTION_LAYER_REQUEST_TIMES, diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 182cad50faf..6af988fa88f 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -699,7 +699,7 @@ pub fn generate_blobs( Ok((bundle, transactions.into())) } -fn static_valid_tx() -> Result, String> { +pub fn static_valid_tx() -> Result, String> { // This is a real transaction hex encoded, but we don't care about the contents of the transaction. let transaction: EthersTransaction = serde_json::from_str( r#"{ diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index 7afeafc321e..77c2410ab1d 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -244,7 +244,7 @@ impl MockExecutionLayer { // TODO: again consider forks let status = self .el - .notify_new_payload(payload.try_into().unwrap()) + .notify_new_payload(payload.to_ref().try_into().unwrap()) .await .unwrap(); assert_eq!(status, PayloadStatus::Valid); diff --git a/beacon_node/execution_layer/src/test_utils/mod.rs b/beacon_node/execution_layer/src/test_utils/mod.rs index b509494703b..425329a520a 100644 --- a/beacon_node/execution_layer/src/test_utils/mod.rs +++ b/beacon_node/execution_layer/src/test_utils/mod.rs @@ -25,8 +25,8 @@ use warp::{http::StatusCode, Filter, Rejection}; use crate::EngineCapabilities; pub use execution_block_generator::{ - generate_blobs, generate_genesis_block, generate_genesis_header, generate_pow_block, Block, - ExecutionBlockGenerator, + generate_blobs, generate_genesis_block, generate_genesis_header, generate_pow_block, + static_valid_tx, Block, ExecutionBlockGenerator, }; pub use hook::Hook; pub use mock_builder::{MockBuilder, Operation}; diff --git a/beacon_node/execution_layer/src/versioned_hashes.rs b/beacon_node/execution_layer/src/versioned_hashes.rs new file mode 100644 index 00000000000..37bd35646d9 --- /dev/null +++ b/beacon_node/execution_layer/src/versioned_hashes.rs @@ -0,0 +1,135 @@ +extern crate alloy_consensus; +extern crate alloy_rlp; +use alloy_consensus::TxEnvelope; +use alloy_rlp::Decodable; +use types::{EthSpec, ExecutionPayloadRef, Hash256, Unsigned, VersionedHash}; + +#[derive(Debug)] +pub enum Error { + DecodingTransaction(String), + LengthMismatch { expected: usize, found: usize }, + VersionHashMismatch { expected: Hash256, found: Hash256 }, +} + +pub fn verify_versioned_hashes( + execution_payload: ExecutionPayloadRef, + expected_versioned_hashes: &[VersionedHash], +) -> Result<(), Error> { + let versioned_hashes = + extract_versioned_hashes_from_transactions::(execution_payload.transactions())?; + if versioned_hashes.len() != expected_versioned_hashes.len() { + return Err(Error::LengthMismatch { + expected: expected_versioned_hashes.len(), + found: versioned_hashes.len(), + }); + } + for (found, expected) in versioned_hashes + .iter() + .zip(expected_versioned_hashes.iter()) + { + if found != expected { + return Err(Error::VersionHashMismatch { + expected: *expected, + found: *found, + }); + } + } + + Ok(()) +} + +pub fn extract_versioned_hashes_from_transactions( + transactions: &types::Transactions, +) -> Result, Error> { + let mut versioned_hashes = Vec::new(); + + for tx in transactions { + match beacon_tx_to_tx_envelope(tx)? { + TxEnvelope::Eip4844(signed_tx_eip4844) => { + versioned_hashes.extend( + signed_tx_eip4844 + .tx() + .blob_versioned_hashes + .iter() + .map(|fb| Hash256::from(fb.0)), + ); + } + // enumerating all variants explicitly to make pattern irrefutable + // in case new types are added in the future which also have blobs + TxEnvelope::Legacy(_) + | TxEnvelope::TaggedLegacy(_) + | TxEnvelope::Eip2930(_) + | TxEnvelope::Eip1559(_) => {} + } + } + + Ok(versioned_hashes) +} + +pub fn beacon_tx_to_tx_envelope( + tx: &types::Transaction, +) -> Result { + let tx_bytes = Vec::from(tx.clone()); + TxEnvelope::decode(&mut tx_bytes.as_slice()) + .map_err(|e| Error::DecodingTransaction(e.to_string())) +} + +#[cfg(test)] +mod test { + use super::*; + use crate::test_utils::static_valid_tx; + use alloy_consensus::{TxKind, TxLegacy}; + + type E = types::MainnetEthSpec; + + #[test] + fn test_decode_static_transaction() { + let valid_tx = static_valid_tx::().expect("should give me known valid transaction"); + let tx_envelope = beacon_tx_to_tx_envelope(&valid_tx).expect("should decode tx"); + let TxEnvelope::Legacy(signed_tx) = tx_envelope else { + panic!("should decode to legacy transaction"); + }; + + assert!(matches!( + signed_tx.tx(), + TxLegacy { + chain_id: Some(0x01), + nonce: 0x15, + gas_price: 0x4a817c800, + to: TxKind::Call(..), + .. + } + )); + } + + #[test] + fn test_extract_versioned_hashes() { + use serde::Deserialize; + + #[derive(Deserialize)] + #[serde(transparent)] + struct TestTransactions( + #[serde(with = "ssz_types::serde_utils::list_of_hex_var_list")] types::Transactions, + ); + + let TestTransactions(raw_transactions): TestTransactions = serde_json::from_str(r#"[ + "0x03f901388501a1f0ff430f843b9aca00843b9aca0082520894e7249813d8ccf6fa95a2203f46a64166073d58878080c002f8c6a0012e98362c814f1724262c0d211a1463418a5f6382a8d457b37a2698afbe7b5ea00100ef985761395dfa8ed5ce91f3f2180b612401909e4cb8f33b90c8a454d9baa0013d45411623b90d90f916e4025ada74b453dd4ca093c017c838367c9de0f801a001753e2af0b1e70e7ef80541355b2a035cc9b2c177418bb2a4402a9b346cf84da0011789b520a8068094a92aa0b04db8d8ef1c6c9818947c5210821732b8744049a0011c4c4f95597305daa5f62bf5f690e37fa11f5de05a95d05cac4e2119e394db80a0ccd86a742af0e042d08cbb35d910ddc24bbc6538f9e53be6620d4b6e1bb77662a01a8bacbc614940ac2f5c23ffc00a122c9f085046883de65c88ab0edb859acb99", + "0x02f9017a8501a1f0ff4382363485012a05f2008512a05f2000830249f094c1b0bc605e2c808aa0867bfc98e51a1fe3e9867f80b901040cc7326300000000000000000000000000000000000000000000000000000000000000a000000000000000000000000000000000000000000000000009445a285baa43e00000000000000000000000000c500931f24edb821cef6e28f7adb33b38578c82000000000000000000000000fc7360b3b28cf4204268a8354dbec60720d155d2000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000009a054a063f0fe7b9c68de8df91aaa5e96c15ab540000000000000000000000000c8d41b8fcc066cdabaf074d78e5153e8ce018a9c080a008e14475c1173cd9f5740c24c08b793f9e16c36c08fa73769db95050e31e3396a019767dcdda26c4a774ca28c9df15d0c20e43bd07bd33ee0f84d6096cb5a1ebed" + ]"#).expect("should get raw transactions"); + let expected_versioned_hashes = vec![ + "0x012e98362c814f1724262c0d211a1463418a5f6382a8d457b37a2698afbe7b5e", + "0x0100ef985761395dfa8ed5ce91f3f2180b612401909e4cb8f33b90c8a454d9ba", + "0x013d45411623b90d90f916e4025ada74b453dd4ca093c017c838367c9de0f801", + "0x01753e2af0b1e70e7ef80541355b2a035cc9b2c177418bb2a4402a9b346cf84d", + "0x011789b520a8068094a92aa0b04db8d8ef1c6c9818947c5210821732b8744049", + "0x011c4c4f95597305daa5f62bf5f690e37fa11f5de05a95d05cac4e2119e394db", + ] + .into_iter() + .map(|tx| Hash256::from_slice(&hex::decode(&tx[2..]).expect("should decode hex"))) + .collect::>(); + + let versioned_hashes = extract_versioned_hashes_from_transactions::(&raw_transactions) + .expect("should get versioned hashes"); + assert_eq!(versioned_hashes, expected_versioned_hashes); + } +} diff --git a/lcli/Dockerfile b/lcli/Dockerfile index aed3628cf32..4f5c3f2972f 100644 --- a/lcli/Dockerfile +++ b/lcli/Dockerfile @@ -1,7 +1,7 @@ # `lcli` requires the full project to be in scope, so this should be built either: # - from the `lighthouse` dir with the command: `docker build -f ./lcli/Dockerflie .` # - from the current directory with the command: `docker build -f ./Dockerfile ../` -FROM rust:1.73.0-bullseye AS builder +FROM rust:1.75.0-bullseye AS builder RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev COPY . lighthouse ARG FEATURES diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index 8517c66c386..b14227e7edf 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -4,7 +4,7 @@ version = "4.6.0" authors = ["Sigma Prime "] edition = { workspace = true } autotests = false -rust-version = "1.73.0" +rust-version = "1.75.0" [features] default = ["slasher-lmdb"] diff --git a/testing/execution_engine_integration/src/test_rig.rs b/testing/execution_engine_integration/src/test_rig.rs index b0701e80a1b..bfa56f63c0d 100644 --- a/testing/execution_engine_integration/src/test_rig.rs +++ b/testing/execution_engine_integration/src/test_rig.rs @@ -381,7 +381,7 @@ impl TestRig { let status = self .ee_a .execution_layer - .notify_new_payload(valid_payload.clone().try_into().unwrap()) + .notify_new_payload(valid_payload.to_ref().try_into().unwrap()) .await .unwrap(); assert_eq!(status, PayloadStatus::Valid); @@ -435,7 +435,7 @@ impl TestRig { let status = self .ee_a .execution_layer - .notify_new_payload(invalid_payload.try_into().unwrap()) + .notify_new_payload(invalid_payload.to_ref().try_into().unwrap()) .await .unwrap(); assert!(matches!( @@ -507,7 +507,7 @@ impl TestRig { let status = self .ee_a .execution_layer - .notify_new_payload(second_payload.clone().try_into().unwrap()) + .notify_new_payload(second_payload.to_ref().try_into().unwrap()) .await .unwrap(); assert_eq!(status, PayloadStatus::Valid); @@ -559,7 +559,7 @@ impl TestRig { let status = self .ee_b .execution_layer - .notify_new_payload(second_payload.clone().try_into().unwrap()) + .notify_new_payload(second_payload.to_ref().try_into().unwrap()) .await .unwrap(); assert!(matches!(status, PayloadStatus::Syncing)); @@ -597,7 +597,7 @@ impl TestRig { let status = self .ee_b .execution_layer - .notify_new_payload(valid_payload.clone().try_into().unwrap()) + .notify_new_payload(valid_payload.to_ref().try_into().unwrap()) .await .unwrap(); assert_eq!(status, PayloadStatus::Valid); @@ -611,7 +611,7 @@ impl TestRig { let status = self .ee_b .execution_layer - .notify_new_payload(second_payload.clone().try_into().unwrap()) + .notify_new_payload(second_payload.to_ref().try_into().unwrap()) .await .unwrap(); assert_eq!(status, PayloadStatus::Valid); From f21472991dbbc9e6e18310a01fc2c4d9ce7c7348 Mon Sep 17 00:00:00 2001 From: realbigsean Date: Sun, 18 Feb 2024 21:22:15 -0500 Subject: [PATCH 18/24] check the da cache and the attester cache in responding to RPC requests (#5138) * check the da cache and the attester cache in responding to RPC requests * use the processing cache instead * update comment * add da cache metrics * rename early attester cache method * Merge branch 'unstable' of https://github.com/sigp/lighthouse into check-da-cache-in-rpc-response * make rustup update run on the runners * Revert "make rustup update run on the runners" This reverts commit d097e9bfa84a13b1d7813c03df38e7756fb0bfc5. --- .../beacon_chain/src/beacon_block_streamer.rs | 33 ++++++------ beacon_node/beacon_chain/src/beacon_chain.rs | 30 +++-------- .../beacon_chain/src/block_verification.rs | 17 +++++++ .../src/block_verification_types.rs | 7 +++ .../src/data_availability_checker.rs | 51 +++++++++++++++---- .../availability_view.rs | 20 ++------ .../overflow_lru_cache.rs | 25 +++++++++ .../processing_cache.rs | 14 +++-- .../state_lru_cache.rs | 3 +- beacon_node/beacon_chain/src/metrics.rs | 43 ++++++++++++++++ .../network_beacon_processor/rpc_methods.rs | 2 +- 11 files changed, 171 insertions(+), 74 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_block_streamer.rs b/beacon_node/beacon_chain/src/beacon_block_streamer.rs index 9312d4511d5..4f4f8ed1fe0 100644 --- a/beacon_node/beacon_chain/src/beacon_block_streamer.rs +++ b/beacon_node/beacon_chain/src/beacon_block_streamer.rs @@ -19,7 +19,7 @@ use types::{ }; #[derive(PartialEq)] -pub enum CheckEarlyAttesterCache { +pub enum CheckCaches { Yes, No, } @@ -385,14 +385,14 @@ impl EngineRequest { pub struct BeaconBlockStreamer { execution_layer: ExecutionLayer, - check_early_attester_cache: CheckEarlyAttesterCache, + check_caches: CheckCaches, beacon_chain: Arc>, } impl BeaconBlockStreamer { pub fn new( beacon_chain: &Arc>, - check_early_attester_cache: CheckEarlyAttesterCache, + check_caches: CheckCaches, ) -> Result { let execution_layer = beacon_chain .execution_layer @@ -402,17 +402,17 @@ impl BeaconBlockStreamer { Ok(Self { execution_layer, - check_early_attester_cache, + check_caches, beacon_chain: beacon_chain.clone(), }) } - fn check_early_attester_cache( - &self, - root: Hash256, - ) -> Option>> { - if self.check_early_attester_cache == CheckEarlyAttesterCache::Yes { - self.beacon_chain.early_attester_cache.get_block(root) + fn check_caches(&self, root: Hash256) -> Option>> { + if self.check_caches == CheckCaches::Yes { + self.beacon_chain + .data_availability_checker + .get_block(&root) + .or(self.beacon_chain.early_attester_cache.get_block(root)) } else { None } @@ -422,10 +422,7 @@ impl BeaconBlockStreamer { let mut db_blocks = Vec::new(); for root in block_roots { - if let Some(cached_block) = self - .check_early_attester_cache(root) - .map(LoadedBeaconBlock::Full) - { + if let Some(cached_block) = self.check_caches(root).map(LoadedBeaconBlock::Full) { db_blocks.push((root, Ok(Some(cached_block)))); continue; } @@ -554,7 +551,7 @@ impl BeaconBlockStreamer { "Using slower fallback method of eth_getBlockByHash()" ); for root in block_roots { - let cached_block = self.check_early_attester_cache(root); + let cached_block = self.check_caches(root); let block_result = if cached_block.is_some() { Ok(cached_block) } else { @@ -682,7 +679,7 @@ impl From for BeaconChainError { #[cfg(test)] mod tests { - use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckEarlyAttesterCache}; + use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckCaches}; use crate::test_utils::{test_spec, BeaconChainHarness, EphemeralHarnessType}; use execution_layer::test_utils::{Block, DEFAULT_ENGINE_CAPABILITIES}; use execution_layer::EngineCapabilities; @@ -804,7 +801,7 @@ mod tests { let start = epoch * slots_per_epoch; let mut epoch_roots = vec![Hash256::zero(); slots_per_epoch]; epoch_roots[..].clone_from_slice(&block_roots[start..(start + slots_per_epoch)]); - let streamer = BeaconBlockStreamer::new(&harness.chain, CheckEarlyAttesterCache::No) + let streamer = BeaconBlockStreamer::new(&harness.chain, CheckCaches::No) .expect("should create streamer"); let (block_tx, mut block_rx) = mpsc::unbounded_channel(); streamer.stream(epoch_roots.clone(), block_tx).await; @@ -945,7 +942,7 @@ mod tests { let start = epoch * slots_per_epoch; let mut epoch_roots = vec![Hash256::zero(); slots_per_epoch]; epoch_roots[..].clone_from_slice(&block_roots[start..(start + slots_per_epoch)]); - let streamer = BeaconBlockStreamer::new(&harness.chain, CheckEarlyAttesterCache::No) + let streamer = BeaconBlockStreamer::new(&harness.chain, CheckCaches::No) .expect("should create streamer"); let (block_tx, mut block_rx) = mpsc::unbounded_channel(); streamer.stream(epoch_roots.clone(), block_tx).await; diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index b05ba01ee7e..20a93e31e8d 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -4,7 +4,7 @@ use crate::attestation_verification::{ VerifiedUnaggregatedAttestation, }; use crate::attester_cache::{AttesterCache, AttesterCacheKey}; -use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckEarlyAttesterCache}; +use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckCaches}; use crate::beacon_proposer_cache::compute_proposer_duties_from_head; use crate::beacon_proposer_cache::BeaconProposerCache; use crate::blob_verification::{GossipBlobError, GossipVerifiedBlob}; @@ -1131,7 +1131,7 @@ impl BeaconChain { /// ## Errors /// /// May return a database error. - pub fn get_blocks_checking_early_attester_cache( + pub fn get_blocks_checking_caches( self: &Arc, block_roots: Vec, executor: &TaskExecutor, @@ -1144,10 +1144,8 @@ impl BeaconChain { >, Error, > { - Ok( - BeaconBlockStreamer::::new(self, CheckEarlyAttesterCache::Yes)? - .launch_stream(block_roots, executor), - ) + Ok(BeaconBlockStreamer::::new(self, CheckCaches::Yes)? + .launch_stream(block_roots, executor)) } pub fn get_blocks( @@ -1163,10 +1161,8 @@ impl BeaconChain { >, Error, > { - Ok( - BeaconBlockStreamer::::new(self, CheckEarlyAttesterCache::No)? - .launch_stream(block_roots, executor), - ) + Ok(BeaconBlockStreamer::::new(self, CheckCaches::No)? + .launch_stream(block_roots, executor)) } pub fn get_blobs_checking_early_attester_cache( @@ -2960,18 +2956,8 @@ impl BeaconChain { unverified_block: B, notify_execution_layer: NotifyExecutionLayer, ) -> Result> { - if let Ok(commitments) = unverified_block - .block() - .message() - .body() - .blob_kzg_commitments() - { - self.data_availability_checker.notify_block_commitments( - unverified_block.block().slot(), - block_root, - commitments.clone(), - ); - }; + self.data_availability_checker + .notify_block(block_root, unverified_block.block_cloned()); let r = self .process_block(block_root, unverified_block, notify_execution_layer, || { Ok(()) diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index e8df5b811ed..ac3d3e3ab80 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -764,6 +764,7 @@ pub trait IntoExecutionPendingBlock: Sized { ) -> Result, BlockSlashInfo>>; fn block(&self) -> &SignedBeaconBlock; + fn block_cloned(&self) -> Arc>; } impl GossipVerifiedBlock { @@ -1017,6 +1018,10 @@ impl IntoExecutionPendingBlock for GossipVerifiedBlock &SignedBeaconBlock { self.block.as_block() } + + fn block_cloned(&self) -> Arc> { + self.block.clone() + } } impl SignatureVerifiedBlock { @@ -1168,6 +1173,10 @@ impl IntoExecutionPendingBlock for SignatureVerifiedBloc fn block(&self) -> &SignedBeaconBlock { self.block.as_block() } + + fn block_cloned(&self) -> Arc> { + self.block.block_cloned() + } } impl IntoExecutionPendingBlock for Arc> { @@ -1198,6 +1207,10 @@ impl IntoExecutionPendingBlock for Arc &SignedBeaconBlock { self } + + fn block_cloned(&self) -> Arc> { + self.clone() + } } impl IntoExecutionPendingBlock for RpcBlock { @@ -1228,6 +1241,10 @@ impl IntoExecutionPendingBlock for RpcBlock fn block(&self) -> &SignedBeaconBlock { self.as_block() } + + fn block_cloned(&self) -> Arc> { + self.block_cloned() + } } impl ExecutionPendingBlock { diff --git a/beacon_node/beacon_chain/src/block_verification_types.rs b/beacon_node/beacon_chain/src/block_verification_types.rs index a6840ed7648..edba7a211cb 100644 --- a/beacon_node/beacon_chain/src/block_verification_types.rs +++ b/beacon_node/beacon_chain/src/block_verification_types.rs @@ -46,6 +46,13 @@ impl RpcBlock { } } + pub fn block_cloned(&self) -> Arc> { + match &self.block { + RpcBlockInner::Block(block) => block.clone(), + RpcBlockInner::BlockAndBlobs(block, _) => block.clone(), + } + } + pub fn blobs(&self) -> Option<&BlobSidecarList> { match &self.block { RpcBlockInner::Block(_) => None, diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 48d505e9e7b..f906032ecd2 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -20,7 +20,7 @@ use std::fmt::Debug; use std::num::NonZeroUsize; use std::sync::Arc; use task_executor::TaskExecutor; -use types::beacon_block_body::{KzgCommitmentOpts, KzgCommitments}; +use types::beacon_block_body::KzgCommitmentOpts; use types::blob_sidecar::{BlobIdentifier, BlobSidecar, FixedBlobSidecarList}; use types::{BlobSidecarList, ChainSpec, Epoch, EthSpec, Hash256, SignedBeaconBlock, Slot}; @@ -192,6 +192,14 @@ impl DataAvailabilityChecker { self.availability_cache.peek_blob(blob_id) } + /// Get a block from the availability cache. Includes any blocks we are currently processing. + pub fn get_block(&self, block_root: &Hash256) -> Option>> { + self.processing_cache + .read() + .get(block_root) + .and_then(|cached| cached.block.clone()) + } + /// Put a list of blobs received via RPC into the availability cache. This performs KZG /// verification on the blobs in the list. pub fn put_rpc_blobs( @@ -344,20 +352,16 @@ impl DataAvailabilityChecker { block.num_expected_blobs() > 0 && self.da_check_required_for_epoch(block.epoch()) } - /// Adds block commitments to the processing cache. These commitments are unverified but caching + /// Adds a block to the processing cache. This block's commitments are unverified but caching /// them here is useful to avoid duplicate downloads of blocks, as well as understanding - /// our blob download requirements. - pub fn notify_block_commitments( - &self, - slot: Slot, - block_root: Hash256, - commitments: KzgCommitments, - ) { + /// our blob download requirements. We will also serve this over RPC. + pub fn notify_block(&self, block_root: Hash256, block: Arc>) { + let slot = block.slot(); self.processing_cache .write() .entry(block_root) .or_insert_with(|| ProcessingComponents::new(slot)) - .merge_block(commitments); + .merge_block(block); } /// Add a single blob commitment to the processing cache. This commitment is unverified but caching @@ -450,6 +454,24 @@ impl DataAvailabilityChecker { pub fn persist_all(&self) -> Result<(), AvailabilityCheckError> { self.availability_cache.write_all_to_disk() } + + /// Collects metrics from the data availability checker. + pub fn metrics(&self) -> DataAvailabilityCheckerMetrics { + DataAvailabilityCheckerMetrics { + processing_cache_size: self.processing_cache.read().len(), + num_store_entries: self.availability_cache.num_store_entries(), + state_cache_size: self.availability_cache.state_cache_size(), + block_cache_size: self.availability_cache.block_cache_size(), + } + } +} + +/// Helper struct to group data availability checker metrics. +pub struct DataAvailabilityCheckerMetrics { + pub processing_cache_size: usize, + pub num_store_entries: usize, + pub state_cache_size: usize, + pub block_cache_size: usize, } pub fn start_availability_cache_maintenance_service( @@ -597,6 +619,15 @@ pub enum MaybeAvailableBlock { }, } +impl MaybeAvailableBlock { + pub fn block_cloned(&self) -> Arc> { + match self { + Self::Available(block) => block.block_cloned(), + Self::AvailabilityPending { block, .. } => block.clone(), + } + } +} + #[derive(Debug, Clone)] pub enum MissingBlobs { /// We know for certain these blobs are missing. diff --git a/beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs b/beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs index 776f81ee545..65093db26bd 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/availability_view.rs @@ -182,9 +182,9 @@ macro_rules! impl_availability_view { impl_availability_view!( ProcessingComponents, - KzgCommitments, + Arc>, KzgCommitment, - block_commitments, + block, blob_commitments ); @@ -212,12 +212,6 @@ pub trait GetCommitment { fn get_commitment(&self) -> &KzgCommitment; } -// These implementations are required to implement `AvailabilityView` for `ProcessingView`. -impl GetCommitments for KzgCommitments { - fn get_commitments(&self) -> KzgCommitments { - self.clone() - } -} impl GetCommitment for KzgCommitment { fn get_commitment(&self) -> &KzgCommitment { self @@ -310,7 +304,7 @@ pub mod tests { } type ProcessingViewSetup = ( - KzgCommitments, + Arc>, FixedVector, ::MaxBlobsPerBlock>, FixedVector, ::MaxBlobsPerBlock>, ); @@ -320,12 +314,6 @@ pub mod tests { valid_blobs: FixedVector>>, ::MaxBlobsPerBlock>, invalid_blobs: FixedVector>>, ::MaxBlobsPerBlock>, ) -> ProcessingViewSetup { - let commitments = block - .message() - .body() - .blob_kzg_commitments() - .unwrap() - .clone(); let blobs = FixedVector::from( valid_blobs .iter() @@ -338,7 +326,7 @@ pub mod tests { .map(|blob_opt| blob_opt.as_ref().map(|blob| blob.kzg_commitment)) .collect::>(), ); - (commitments, blobs, invalid_blobs) + (Arc::new(block), blobs, invalid_blobs) } type PendingComponentsSetup = ( diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 34c9bc76f6e..80cbc6c8990 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -363,6 +363,16 @@ impl Critical { } } } + + /// Returns the number of pending component entries in memory. + pub fn num_blocks(&self) -> usize { + self.in_memory.len() + } + + /// Returns the number of entries that have overflowed to disk. + pub fn num_store_entries(&self) -> usize { + self.store_keys.len() + } } /// This is the main struct for this module. Outside methods should @@ -671,6 +681,21 @@ impl OverflowLRUCache { pub fn state_lru_cache(&self) -> &StateLRUCache { &self.state_cache } + + /// Number of states stored in memory in the cache. + pub fn state_cache_size(&self) -> usize { + self.state_cache.lru_cache().read().len() + } + + /// Number of pending component entries in memory in the cache. + pub fn block_cache_size(&self) -> usize { + self.critical.read().num_blocks() + } + + /// Returns the number of entries in the cache that have overflowed to disk. + pub fn num_store_entries(&self) -> usize { + self.critical.read().num_store_entries() + } } impl ssz::Encode for OverflowKey { diff --git a/beacon_node/beacon_chain/src/data_availability_checker/processing_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/processing_cache.rs index 969034c6570..af94803dcfb 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/processing_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/processing_cache.rs @@ -1,8 +1,9 @@ use crate::data_availability_checker::AvailabilityView; use std::collections::hash_map::Entry; use std::collections::HashMap; -use types::beacon_block_body::{KzgCommitmentOpts, KzgCommitments}; -use types::{EthSpec, Hash256, Slot}; +use std::sync::Arc; +use types::beacon_block_body::KzgCommitmentOpts; +use types::{EthSpec, Hash256, SignedBeaconBlock, Slot}; /// This cache is used only for gossip blocks/blobs and single block/blob lookups, to give req/resp /// a view of what we have and what we require. This cache serves a slightly different purpose than @@ -37,6 +38,9 @@ impl ProcessingCache { } roots_missing_components } + pub fn len(&self) -> usize { + self.processing_cache.len() + } } #[derive(Debug, Clone)] @@ -45,7 +49,7 @@ pub struct ProcessingComponents { /// Blobs required for a block can only be known if we have seen the block. So `Some` here /// means we've seen it, a `None` means we haven't. The `kzg_commitments` value helps us figure /// out whether incoming blobs actually match the block. - pub block_commitments: Option>, + pub block: Option>>, /// `KzgCommitments` for blobs are always known, even if we haven't seen the block. See /// `AvailabilityView`'s trait definition for more details. pub blob_commitments: KzgCommitmentOpts, @@ -55,7 +59,7 @@ impl ProcessingComponents { pub fn new(slot: Slot) -> Self { Self { slot, - block_commitments: None, + block: None, blob_commitments: KzgCommitmentOpts::::default(), } } @@ -67,7 +71,7 @@ impl ProcessingComponents { pub fn empty(_block_root: Hash256) -> Self { Self { slot: Slot::new(0), - block_commitments: None, + block: None, blob_commitments: KzgCommitmentOpts::::default(), } } diff --git a/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs index bd125a7f42a..35c114db542 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/state_lru_cache.rs @@ -190,8 +190,7 @@ impl StateLRUCache { }) } - /// returns the state cache for inspection in tests - #[cfg(test)] + /// returns the state cache for inspection pub fn lru_cache(&self) -> &RwLock>> { &self.states } diff --git a/beacon_node/beacon_chain/src/metrics.rs b/beacon_node/beacon_chain/src/metrics.rs index 24c05e01f22..abac2c80e7f 100644 --- a/beacon_node/beacon_chain/src/metrics.rs +++ b/beacon_node/beacon_chain/src/metrics.rs @@ -1129,6 +1129,31 @@ lazy_static! { Ok(vec![0.1, 0.2, 0.3,0.4,0.5,0.75,1.0,1.25,1.5,1.75,2.0,2.5,3.0,3.5,4.0,5.0,6.0,7.0,8.0,9.0,10.0,15.0,20.0]) ); + + /* + * Data Availability cache metrics + */ + pub static ref DATA_AVAILABILITY_PROCESSING_CACHE_SIZE: Result = + try_create_int_gauge( + "data_availability_processing_cache_size", + "Number of entries in the data availability processing cache." + ); + pub static ref DATA_AVAILABILITY_OVERFLOW_MEMORY_BLOCK_CACHE_SIZE: Result = + try_create_int_gauge( + "data_availability_overflow_memory_block_cache_size", + "Number of entries in the data availability overflow block memory cache." + ); + pub static ref DATA_AVAILABILITY_OVERFLOW_MEMORY_STATE_CACHE_SIZE: Result = + try_create_int_gauge( + "data_availability_overflow_memory_state_cache_size", + "Number of entries in the data availability overflow state memory cache." + ); + pub static ref DATA_AVAILABILITY_OVERFLOW_STORE_CACHE_SIZE: Result = + try_create_int_gauge( + "data_availability_overflow_store_cache_size", + "Number of entries in the data availability overflow store cache." + ); + /* * light_client server metrics */ @@ -1171,6 +1196,24 @@ pub fn scrape_for_metrics(beacon_chain: &BeaconChain) { ) } + let da_checker_metrics = beacon_chain.data_availability_checker.metrics(); + set_gauge_by_usize( + &DATA_AVAILABILITY_PROCESSING_CACHE_SIZE, + da_checker_metrics.processing_cache_size, + ); + set_gauge_by_usize( + &DATA_AVAILABILITY_OVERFLOW_MEMORY_BLOCK_CACHE_SIZE, + da_checker_metrics.block_cache_size, + ); + set_gauge_by_usize( + &DATA_AVAILABILITY_OVERFLOW_MEMORY_STATE_CACHE_SIZE, + da_checker_metrics.state_cache_size, + ); + set_gauge_by_usize( + &DATA_AVAILABILITY_OVERFLOW_STORE_CACHE_SIZE, + da_checker_metrics.num_store_entries, + ); + if let Some((size, num_lookups)) = beacon_chain.pre_finalization_block_cache.metrics() { set_gauge_by_usize(&PRE_FINALIZATION_BLOCK_CACHE_SIZE, size); set_gauge_by_usize(&PRE_FINALIZATION_BLOCK_LOOKUP_COUNT, num_lookups); diff --git a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs index a731dea7c19..66c98ff3b84 100644 --- a/beacon_node/network/src/network_beacon_processor/rpc_methods.rs +++ b/beacon_node/network/src/network_beacon_processor/rpc_methods.rs @@ -140,7 +140,7 @@ impl NetworkBeaconProcessor { let requested_blocks = request.block_roots().len(); let mut block_stream = match self .chain - .get_blocks_checking_early_attester_cache(request.block_roots().to_vec(), &executor) + .get_blocks_checking_caches(request.block_roots().to_vec(), &executor) { Ok(block_stream) => block_stream, Err(e) => return error!(self.log, "Error getting block stream"; "error" => ?e), From e22c9eed8f9de26f14d78989d363b20009a20875 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Mon, 19 Feb 2024 13:22:19 +1100 Subject: [PATCH 19/24] Add Deneb fork epoch for Gnosis (#5242) * Add Deneb fork epoch for Gnosis * Add deneb constants * Update common/eth2_network_config/built_in_network_configs/gnosis/config.yaml Co-authored-by: realbigsean * Adjust `min_epochs_for_block_requests` * Change `MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT` * Fix chain spec `max_per_epoch_activation_churn_limit` * Fix chain spec values again --- .../built_in_network_configs/gnosis/config.yaml | 14 ++++++++++++-- consensus/types/src/chain_spec.rs | 8 ++++---- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml b/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml index 18a08ed12d7..2311d6db0f9 100644 --- a/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml +++ b/common/eth2_network_config/built_in_network_configs/gnosis/config.yaml @@ -41,7 +41,7 @@ CAPELLA_FORK_VERSION: 0x03000064 CAPELLA_FORK_EPOCH: 648704 # Deneb DENEB_FORK_VERSION: 0x04000064 -DENEB_FORK_EPOCH: 18446744073709551615 +DENEB_FORK_EPOCH: 889856 # 2024-03-11T18:30:20.000Z # Sharding SHARDING_FORK_VERSION: 0x03000064 SHARDING_FORK_EPOCH: 18446744073709551615 @@ -75,7 +75,7 @@ EJECTION_BALANCE: 16000000000 # 2**2 (= 4) MIN_PER_EPOCH_CHURN_LIMIT: 4 # 2**3 (= 8) -MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 8 +MAX_PER_EPOCH_ACTIVATION_CHURN_LIMIT: 2 # 2**12 (= 4096) CHURN_LIMIT_QUOTIENT: 4096 @@ -106,3 +106,13 @@ ATTESTATION_SUBNET_COUNT: 64 ATTESTATION_SUBNET_EXTRA_BITS: 0 ATTESTATION_SUBNET_PREFIX_BITS: 6 ATTESTATION_SUBNET_SHUFFLING_PREFIX_BITS: 3 + +# Deneb +# `2**7` (=128) +MAX_REQUEST_BLOCKS_DENEB: 128 +# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK +MAX_REQUEST_BLOB_SIDECARS: 768 +# `2**14` (= 16384 epochs, ~15 days) +MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 16384 +# `6` +BLOB_SIDECAR_SUBNET_COUNT: 6 diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index c32c67fa330..88f989d2a5a 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -809,7 +809,7 @@ impl ChainSpec { max_committees_per_slot: 64, target_committee_size: 128, min_per_epoch_churn_limit: 4, - max_per_epoch_activation_churn_limit: 8, + max_per_epoch_activation_churn_limit: 2, churn_limit_quotient: 4_096, shuffle_round_count: 90, min_genesis_active_validator_count: 4_096, @@ -944,7 +944,7 @@ impl ChainSpec { * Deneb hard fork params */ deneb_fork_version: [0x04, 0x00, 0x00, 0x64], - deneb_fork_epoch: None, + deneb_fork_epoch: Some(Epoch::new(889856)), /* * Network specific @@ -958,7 +958,7 @@ impl ChainSpec { target_aggregators_per_committee: 16, epochs_per_subnet_subscription: default_epochs_per_subnet_subscription(), gossip_max_size: default_gossip_max_size(), - min_epochs_for_block_requests: default_min_epochs_for_block_requests(), + min_epochs_for_block_requests: 33024, max_chunk_size: default_max_chunk_size(), ttfb_timeout: default_ttfb_timeout(), resp_timeout: default_resp_timeout(), @@ -975,7 +975,7 @@ impl ChainSpec { */ max_request_blocks_deneb: default_max_request_blocks_deneb(), max_request_blob_sidecars: default_max_request_blob_sidecars(), - min_epochs_for_blob_sidecars_requests: default_min_epochs_for_blob_sidecars_requests(), + min_epochs_for_blob_sidecars_requests: 16384, blob_sidecar_subnet_count: default_blob_sidecar_subnet_count(), /* From c9702cb0a1a0307d2ccf8f8d3d072ef94f3d4c59 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Mon, 19 Feb 2024 13:22:23 +1100 Subject: [PATCH 20/24] Download checkpoint blobs during checkpoint sync (#5252) * MVP implementation (untested) * update store checkpoint sync test * update cli help * Merge pull request #5253 from realbigsean/checkpoint-blobs-sean Checkpoint blobs sean * Warn only if blobs are missing from server * Merge remote-tracking branch 'origin/unstable' into checkpoint-blobs * Verify checkpoint blobs * Move blob verification earlier --- Cargo.lock | 1 + beacon_node/beacon_chain/src/builder.rs | 39 +++++++++++++-- beacon_node/beacon_chain/tests/store_tests.rs | 19 ++++++- beacon_node/client/Cargo.toml | 1 + beacon_node/client/src/builder.rs | 49 +++++++++++++++++-- beacon_node/client/src/config.rs | 1 + beacon_node/src/cli.rs | 9 ++++ beacon_node/src/config.rs | 5 +- book/src/help_bn.md | 3 ++ consensus/types/src/beacon_block_body.rs | 6 +++ 10 files changed, 122 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1c32fed0874..5885a7387f2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1495,6 +1495,7 @@ dependencies = [ "eth1", "eth2", "eth2_config", + "ethereum_ssz", "execution_layer", "futures", "genesis", diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index abd0e6b6a82..c75c3f695b3 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -39,8 +39,8 @@ use std::time::Duration; use store::{Error as StoreError, HotColdDB, ItemStore, KeyValueStoreOp}; use task_executor::{ShutdownReason, TaskExecutor}; use types::{ - BeaconBlock, BeaconState, ChainSpec, Checkpoint, Epoch, EthSpec, Graffiti, Hash256, Signature, - SignedBeaconBlock, Slot, + BeaconBlock, BeaconState, BlobSidecarList, ChainSpec, Checkpoint, Epoch, EthSpec, Graffiti, + Hash256, Signature, SignedBeaconBlock, Slot, }; /// An empty struct used to "witness" all the `BeaconChainTypes` traits. It has no user-facing @@ -432,6 +432,7 @@ where mut self, mut weak_subj_state: BeaconState, weak_subj_block: SignedBeaconBlock, + weak_subj_blobs: Option>, genesis_state: BeaconState, ) -> Result { let store = self @@ -490,6 +491,29 @@ where )); } + // Verify that blobs (if provided) match the block. + if let Some(blobs) = &weak_subj_blobs { + let commitments = weak_subj_block + .message() + .body() + .blob_kzg_commitments() + .map_err(|e| format!("Blobs provided but block does not reference them: {e:?}"))?; + if blobs.len() != commitments.len() { + return Err(format!( + "Wrong number of blobs, expected: {}, got: {}", + commitments.len(), + blobs.len() + )); + } + if commitments + .iter() + .zip(blobs.iter()) + .any(|(commitment, blob)| *commitment != blob.kzg_commitment) + { + return Err("Checkpoint blob does not match block commitment".into()); + } + } + // Set the store's split point *before* storing genesis so that genesis is stored // immediately in the freezer DB. store.set_split(weak_subj_slot, weak_subj_state_root, weak_subj_block_root); @@ -511,14 +535,19 @@ where .do_atomically(block_root_batch) .map_err(|e| format!("Error writing frozen block roots: {e:?}"))?; - // Write the state and block non-atomically, it doesn't matter if they're forgotten + // Write the state, block and blobs non-atomically, it doesn't matter if they're forgotten // about on a crash restart. store .put_state(&weak_subj_state_root, &weak_subj_state) - .map_err(|e| format!("Failed to store weak subjectivity state: {:?}", e))?; + .map_err(|e| format!("Failed to store weak subjectivity state: {e:?}"))?; store .put_block(&weak_subj_block_root, weak_subj_block.clone()) - .map_err(|e| format!("Failed to store weak subjectivity block: {:?}", e))?; + .map_err(|e| format!("Failed to store weak subjectivity block: {e:?}"))?; + if let Some(blobs) = weak_subj_blobs { + store + .put_blobs(&weak_subj_block_root, blobs) + .map_err(|e| format!("Failed to store weak subjectivity blobs: {e:?}"))?; + } // Stage the database's metadata fields for atomic storage when `build` is called. // This prevents the database from restarting in an inconsistent state if the anchor diff --git a/beacon_node/beacon_chain/tests/store_tests.rs b/beacon_node/beacon_chain/tests/store_tests.rs index ffd1b843a18..ff201729821 100644 --- a/beacon_node/beacon_chain/tests/store_tests.rs +++ b/beacon_node/beacon_chain/tests/store_tests.rs @@ -2396,6 +2396,7 @@ async fn weak_subjectivity_sync_test(slots: Vec, checkpoint_slot: Slot) { .get_full_block(&wss_block_root) .unwrap() .unwrap(); + let wss_blobs_opt = harness.chain.store.get_blobs(&wss_block_root).unwrap(); let wss_state = full_store .get_state(&wss_state_root, Some(checkpoint_slot)) .unwrap() @@ -2438,7 +2439,12 @@ async fn weak_subjectivity_sync_test(slots: Vec, checkpoint_slot: Slot) { .custom_spec(test_spec::()) .task_executor(harness.chain.task_executor.clone()) .logger(log.clone()) - .weak_subjectivity_state(wss_state, wss_block.clone(), genesis_state) + .weak_subjectivity_state( + wss_state, + wss_block.clone(), + wss_blobs_opt.clone(), + genesis_state, + ) .unwrap() .store_migrator_config(MigratorConfig::default().blocking()) .dummy_eth1_backend() @@ -2456,6 +2462,17 @@ async fn weak_subjectivity_sync_test(slots: Vec, checkpoint_slot: Slot) { .expect("should build"); let beacon_chain = Arc::new(beacon_chain); + let wss_block_root = wss_block.canonical_root(); + let store_wss_block = harness + .chain + .get_block(&wss_block_root) + .await + .unwrap() + .unwrap(); + let store_wss_blobs_opt = beacon_chain.store.get_blobs(&wss_block_root).unwrap(); + + assert_eq!(store_wss_block, wss_block); + assert_eq!(store_wss_blobs_opt, wss_blobs_opt); // Apply blocks forward to reach head. let chain_dump = harness.chain.chain_dump().unwrap(); diff --git a/beacon_node/client/Cargo.toml b/beacon_node/client/Cargo.toml index 0160cad4b20..03cbcc9ff7f 100644 --- a/beacon_node/client/Cargo.toml +++ b/beacon_node/client/Cargo.toml @@ -45,3 +45,4 @@ monitoring_api = { workspace = true } execution_layer = { workspace = true } beacon_processor = { workspace = true } num_cpus = { workspace = true } +ethereum_ssz = { workspace = true } diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index de6e08cc37a..558e5cbc84f 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -36,6 +36,7 @@ use network::{NetworkConfig, NetworkSenders, NetworkService}; use slasher::Slasher; use slasher_service::SlasherService; use slog::{debug, info, warn, Logger}; +use ssz::Decode; use std::net::TcpListener; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -44,7 +45,7 @@ use std::time::{SystemTime, UNIX_EPOCH}; use timer::spawn_timer; use tokio::sync::oneshot; use types::{ - test_utils::generate_deterministic_keypairs, BeaconState, ChainSpec, EthSpec, + test_utils::generate_deterministic_keypairs, BeaconState, BlobSidecarList, ChainSpec, EthSpec, ExecutionBlockHash, Hash256, SignedBeaconBlock, }; @@ -319,6 +320,7 @@ where ClientGenesis::WeakSubjSszBytes { anchor_state_bytes, anchor_block_bytes, + anchor_blobs_bytes, } => { info!(context.log(), "Starting checkpoint sync"); if config.chain.genesis_backfill { @@ -332,10 +334,25 @@ where .map_err(|e| format!("Unable to parse weak subj state SSZ: {:?}", e))?; let anchor_block = SignedBeaconBlock::from_ssz_bytes(&anchor_block_bytes, &spec) .map_err(|e| format!("Unable to parse weak subj block SSZ: {:?}", e))?; + let anchor_blobs = if anchor_block.message().body().has_blobs() { + let anchor_blobs_bytes = anchor_blobs_bytes + .ok_or("Blobs for checkpoint must be provided using --checkpoint-blobs")?; + Some( + BlobSidecarList::from_ssz_bytes(&anchor_blobs_bytes) + .map_err(|e| format!("Unable to parse weak subj blobs SSZ: {e:?}"))?, + ) + } else { + None + }; let genesis_state = genesis_state(&runtime_context, &config, log).await?; builder - .weak_subjectivity_state(anchor_state, anchor_block, genesis_state) + .weak_subjectivity_state( + anchor_state, + anchor_block, + anchor_blobs, + genesis_state, + ) .map(|v| (v, None))? } ClientGenesis::CheckpointSyncUrl { url } => { @@ -430,9 +447,33 @@ where e => format!("Error fetching finalized block from remote: {:?}", e), })? .ok_or("Finalized block missing from remote, it returned 404")?; + let block_root = block.canonical_root(); debug!(context.log(), "Downloaded finalized block"); + let blobs = if block.message().body().has_blobs() { + debug!(context.log(), "Downloading finalized blobs"); + if let Some(response) = remote + .get_blobs::(BlockId::Root(block_root), None) + .await + .map_err(|e| format!("Error fetching finalized blobs from remote: {e:?}"))? + { + debug!(context.log(), "Downloaded finalized blobs"); + Some(response.data) + } else { + warn!( + context.log(), + "Checkpoint server is missing blobs"; + "block_root" => %block_root, + "hint" => "use a different URL or ask the provider to update", + "impact" => "db will be slightly corrupt until these blobs are pruned", + ); + None + } + } else { + None + }; + let genesis_state = genesis_state(&runtime_context, &config, log).await?; info!( @@ -440,7 +481,7 @@ where "Loaded checkpoint block and state"; "block_slot" => block.slot(), "state_slot" => state.slot(), - "block_root" => ?block.canonical_root(), + "block_root" => ?block_root, ); let service = @@ -468,7 +509,7 @@ where }); builder - .weak_subjectivity_state(state, block, genesis_state) + .weak_subjectivity_state(state, block, blobs, genesis_state) .map(|v| (v, service))? } ClientGenesis::DepositContract => { diff --git a/beacon_node/client/src/config.rs b/beacon_node/client/src/config.rs index 275f9998640..197f21c64ed 100644 --- a/beacon_node/client/src/config.rs +++ b/beacon_node/client/src/config.rs @@ -35,6 +35,7 @@ pub enum ClientGenesis { WeakSubjSszBytes { anchor_state_bytes: Vec, anchor_block_bytes: Vec, + anchor_blobs_bytes: Option>, }, CheckpointSyncUrl { url: SensitiveUrl, diff --git a/beacon_node/src/cli.rs b/beacon_node/src/cli.rs index 4ed60d27b0f..fa4edc34d22 100644 --- a/beacon_node/src/cli.rs +++ b/beacon_node/src/cli.rs @@ -939,6 +939,15 @@ pub fn cli_app<'a, 'b>() -> App<'a, 'b> { .takes_value(true) .requires("checkpoint-state") ) + .arg( + Arg::with_name("checkpoint-blobs") + .long("checkpoint-blobs") + .help("Set the checkpoint blobs to start syncing from. Must be aligned and match \ + --checkpoint-block. Using --checkpoint-sync-url instead is recommended.") + .value_name("BLOBS_SSZ") + .takes_value(true) + .requires("checkpoint-block") + ) .arg( Arg::with_name("checkpoint-sync-url") .long("checkpoint-sync-url") diff --git a/beacon_node/src/config.rs b/beacon_node/src/config.rs index 1699c51784f..fc6132f8c9b 100644 --- a/beacon_node/src/config.rs +++ b/beacon_node/src/config.rs @@ -512,9 +512,10 @@ pub fn get_config( client_config.genesis = if eth2_network_config.genesis_state_is_known() { // Set up weak subjectivity sync, or start from the hardcoded genesis state. - if let (Some(initial_state_path), Some(initial_block_path)) = ( + if let (Some(initial_state_path), Some(initial_block_path), opt_initial_blobs_path) = ( cli_args.value_of("checkpoint-state"), cli_args.value_of("checkpoint-block"), + cli_args.value_of("checkpoint-blobs"), ) { let read = |path: &str| { use std::fs::File; @@ -530,10 +531,12 @@ pub fn get_config( let anchor_state_bytes = read(initial_state_path)?; let anchor_block_bytes = read(initial_block_path)?; + let anchor_blobs_bytes = opt_initial_blobs_path.map(read).transpose()?; ClientGenesis::WeakSubjSszBytes { anchor_state_bytes, anchor_block_bytes, + anchor_blobs_bytes, } } else if let Some(remote_bn_url) = cli_args.value_of("checkpoint-sync-url") { let url = SensitiveUrl::parse(remote_bn_url) diff --git a/book/src/help_bn.md b/book/src/help_bn.md index 3d2124964b2..b2a922020f5 100644 --- a/book/src/help_bn.md +++ b/book/src/help_bn.md @@ -179,6 +179,9 @@ OPTIONS: --builder-user-agent The HTTP user agent to send alongside requests to the builder URL. The default is Lighthouse's version string. + --checkpoint-blobs + Set the checkpoint blobs to start syncing from. Must be aligned and match --checkpoint-block. Using + --checkpoint-sync-url instead is recommended. --checkpoint-block Set a checkpoint block to start syncing from. Must be aligned and match --checkpoint-state. Using --checkpoint-sync-url instead is recommended. diff --git a/consensus/types/src/beacon_block_body.rs b/consensus/types/src/beacon_block_body.rs index 146dff895c8..b0a942d74a4 100644 --- a/consensus/types/src/beacon_block_body.rs +++ b/consensus/types/src/beacon_block_body.rs @@ -176,6 +176,12 @@ impl<'a, T: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, T, } } } + + /// Return `true` if this block body has a non-zero number of blobs. + pub fn has_blobs(self) -> bool { + self.blob_kzg_commitments() + .map_or(false, |blobs| !blobs.is_empty()) + } } impl<'a, T: EthSpec, Payload: AbstractExecPayload> BeaconBlockBodyRef<'a, T, Payload> { From 4d625951b823f7365dcaeb4e5c8f15263af43eb0 Mon Sep 17 00:00:00 2001 From: Eitan Seri-Levi Date: Mon, 19 Feb 2024 07:17:58 +0200 Subject: [PATCH 21/24] Deprecate env_log flag in tracing layer (#5228) * deprecate terminal logs file in tracing layer * sink writer --- common/logging/src/lib.rs | 8 ++------ lighthouse/src/main.rs | 4 +--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/common/logging/src/lib.rs b/common/logging/src/lib.rs index caf3e1d2fb2..3a5a5209b02 100644 --- a/common/logging/src/lib.rs +++ b/common/logging/src/lib.rs @@ -223,7 +223,7 @@ impl TimeLatch { } } -pub fn create_tracing_layer(base_tracing_log_path: PathBuf, turn_on_terminal_logs: bool) { +pub fn create_tracing_layer(base_tracing_log_path: PathBuf) { let filter_layer = match tracing_subscriber::EnvFilter::try_from_default_env() .or_else(|_| tracing_subscriber::EnvFilter::try_new("warn")) { @@ -268,11 +268,7 @@ pub fn create_tracing_layer(base_tracing_log_path: PathBuf, turn_on_terminal_log if let Err(e) = tracing_subscriber::fmt() .with_env_filter(filter_layer) - .with_writer(move || { - tracing_subscriber::fmt::writer::OptionalWriter::::from( - turn_on_terminal_logs.then(std::io::stdout), - ) - }) + .with_writer(std::io::sink) .finish() .with(MetricsLayer) .with(custom_layer) diff --git a/lighthouse/src/main.rs b/lighthouse/src/main.rs index 022b208359d..d646b9764cd 100644 --- a/lighthouse/src/main.rs +++ b/lighthouse/src/main.rs @@ -556,9 +556,7 @@ fn run( let path = tracing_log_path.clone().unwrap(); - let turn_on_terminal_logs = matches.is_present("env_log"); - - logging::create_tracing_layer(path, turn_on_terminal_logs); + logging::create_tracing_layer(path); // Allow Prometheus to export the time at which the process was started. metrics::expose_process_start_time(&log); From a229b5272354cee313c6267456f724f3b9b92ead Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Oliveira?= Date: Mon, 19 Feb 2024 07:16:01 +0000 Subject: [PATCH 22/24] Deactivate RPC Connection Handler after goodbye message is sent (#5250) * Deactivate RPC Connection Handler after goodbye message is sent * nit: use to_string instead of format * Merge branch 'unstable' of https://github.com/sigp/lighthouse into rpc-shutdown-improvement * clippy * Fix cargo.lock * Merge latest unstable --- Cargo.lock | 376 +++++++++--------- .../lighthouse_network/src/rpc/handler.rs | 32 +- beacon_node/lighthouse_network/src/rpc/mod.rs | 17 +- 3 files changed, 215 insertions(+), 210 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 5885a7387f2..46fd981aa77 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -115,9 +115,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac1f845298e95f983ff1944b728ae08b8cebab80d684f0a832ed0fc74dfa27e2" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ "cfg-if", "cipher 0.4.4", @@ -126,14 +126,14 @@ dependencies = [ [[package]] name = "aes-gcm" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc3be92e19a7ef47457b8e6f90707e12b6ac5d20c6f3866584fa3be0787d839f" +checksum = "df5f85a83a7d8b0442b6aa7b504b8212c1733da07b98aae43d4bc21b2cb3cdf6" dependencies = [ "aead 0.4.3", "aes 0.7.5", "cipher 0.3.0", - "ctr 0.7.0", + "ctr 0.8.0", "ghash 0.4.4", "subtle", ] @@ -145,7 +145,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1" dependencies = [ "aead 0.5.2", - "aes 0.8.3", + "aes 0.8.4", "cipher 0.4.4", "ctr 0.9.2", "ghash 0.5.0", @@ -154,9 +154,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c3a9648d43b9cd48db467b3f87fdd6e146bcc88ab0180006cef2179fe11d01" +checksum = "42cd52102d3df161c77a887b608d7a4897d7cc112886a9537b738a887a03aaff" dependencies = [ "cfg-if", "once_cell", @@ -226,9 +226,9 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4b6fb2b432ff223d513db7f908937f63c252bee0af9b82bfd25b0a5dd1eb0d8" +checksum = "ef197eb250c64962003cb08b90b17f0882c192f4a6f2f544809d424fd7cb0e7d" dependencies = [ "alloy-rlp", "bytes", @@ -265,7 +265,7 @@ checksum = "1a047897373be4bbb0224c1afdabca92648dc57a9c9ef6e7b0be3aff7a859c83" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -512,13 +512,13 @@ dependencies = [ [[package]] name = "async-channel" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ca33f4bc4ed1babef42cad36cc1f51fa88be00420404e5b1e80ab1b18f7678c" +checksum = "f28243a43d821d11341ab73c80bed182dc015c514b951616cf79bd4af39af0c3" dependencies = [ "concurrent-queue", - "event-listener 4.0.3", - "event-listener-strategy", + "event-listener 5.1.0", + "event-listener-strategy 0.5.0", "futures-core", "pin-project-lite", ] @@ -543,7 +543,7 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05b1b633a2115cd122d73b955eadd9916c18c8f510ec9cd1686404c60ad1c29c" dependencies = [ - "async-channel 2.1.1", + "async-channel 2.2.0", "async-executor", "async-io 2.3.1", "async-lock 3.3.0", @@ -584,8 +584,8 @@ dependencies = [ "futures-io", "futures-lite 2.2.0", "parking", - "polling 3.3.2", - "rustix 0.38.30", + "polling 3.5.0", + "rustix 0.38.31", "slab", "tracing", "windows-sys 0.52.0", @@ -607,7 +607,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d034b430882f8381900d3fe6f0aaa3ad94f2cb4ac519b429692a1bc2dda4ae7b" dependencies = [ "event-listener 4.0.3", - "event-listener-strategy", + "event-listener-strategy 0.4.0", "pin-project-lite", ] @@ -624,7 +624,7 @@ dependencies = [ "cfg-if", "event-listener 3.1.0", "futures-lite 1.13.0", - "rustix 0.38.30", + "rustix 0.38.31", "windows-sys 0.48.0", ] @@ -640,7 +640,7 @@ dependencies = [ "cfg-if", "futures-core", "futures-io", - "rustix 0.38.30", + "rustix 0.38.31", "signal-hook-registry", "slab", "windows-sys 0.48.0", @@ -687,7 +687,7 @@ checksum = "c980ee35e870bd1a4d2c8294d4c04d0499e67bca1e4b5cefcc693c2fa00caea9" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -763,7 +763,7 @@ checksum = "823b8bb275161044e2ac7a25879cb3e2480cb403e3943022c7c769c599b756aa" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -1047,7 +1047,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.48", + "syn 2.0.49", "which", ] @@ -1142,7 +1142,7 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a37913e8dc4ddcc604f0c6d3bf2887c995153af3611de9e23c352b44c1b9118" dependencies = [ - "async-channel 2.1.1", + "async-channel 2.2.0", "async-lock 3.3.0", "async-task", "fastrand 2.0.1", @@ -1246,9 +1246,9 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.14.0" +version = "3.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f30e7476521f6f8af1a1c4c0b8cc94f0bee37d91763d0ca2665f299b6cd8aec" +checksum = "d32a994c2b3ca201d9b263612a374263f05e7adde37c4707f693dcd375076d1f" [[package]] name = "byte-slice-cast" @@ -1332,9 +1332,9 @@ dependencies = [ [[package]] name = "cargo-platform" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ceed8ef69d8518a5dda55c07425450b58a4e1946f4951eab6d7191ee86c2443d" +checksum = "694c8807f2ae16faecc43dc17d74b3eb042482789fd0eb64b39a2e04e087053f" dependencies = [ "serde", ] @@ -1410,9 +1410,9 @@ dependencies = [ [[package]] name = "chrono" -version = "0.4.33" +version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f13690e35a5e4ace198e7beea2895d29f3a9cc55015fcebe6336bd2010af9eb" +checksum = "5bc015644b92d5890fab7489e49d21f879d5c990186827d42ec511919404f38b" dependencies = [ "android-tzdata", "iana-time-zone", @@ -1562,9 +1562,9 @@ dependencies = [ [[package]] name = "const-hex" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5104de16b218eddf8e34ffe2f86f74bfa4e61e95a1b89732fccf6325efd0557" +checksum = "18d59688ad0945eaf6b84cb44fedbe93484c81b48970e98f09db8a22832d7961" dependencies = [ "cfg-if", "cpufeatures", @@ -1627,9 +1627,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] @@ -1757,9 +1757,9 @@ dependencies = [ [[package]] name = "crypto-mac" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25fab6889090c8133f3deb8f73ba3c65a7f456f66436fc012a1b1e272b1e103e" +checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714" dependencies = [ "generic-array", "subtle", @@ -1786,15 +1786,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "ctr" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a232f92a03f37dd7d7dd2adc67166c77e9cd88de5b019b9a9eecfaeaf7bfd481" -dependencies = [ - "cipher 0.3.0", -] - [[package]] name = "ctr" version = "0.8.0" @@ -1848,7 +1839,7 @@ checksum = "f46882e17999c6cc590af592290432be3bce0428cb0d5f8b6715e4dc7b383eb3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -2050,7 +2041,7 @@ checksum = "67e77553c4162a157adbf834ebae5b415acbecbeafc7a74b0e886657506a7611" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -2089,7 +2080,7 @@ dependencies = [ "diesel_table_macro_syntax", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -2109,7 +2100,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc5557efc453706fed5e4fa85006fe9817c224c3f480a34c7e5959fd700921c5" dependencies = [ - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -2190,7 +2181,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bac33cb3f99889a57e56a8c6ccb77aaf0cfc7787602b7af09783f736d77314e1" dependencies = [ "aes 0.7.5", - "aes-gcm 0.9.2", + "aes-gcm 0.9.4", "arrayvec", "delay_map", "enr", @@ -2222,7 +2213,7 @@ checksum = "487585f4d0c6655fe74905e2504d8ad6908e4db67f744eb140876906c2f3175d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -2275,9 +2266,9 @@ dependencies = [ [[package]] name = "ed25519-dalek" -version = "2.1.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f628eaec48bfd21b865dc2950cfa014450c01d2fa2b69a86c2fd5844ec523c0" +checksum = "4a3daa8e81a3963a60642bcc1f90a670680bd4a77535faa384e9d1c79d620871" dependencies = [ "curve25519-dalek", "ed25519", @@ -2325,9 +2316,9 @@ dependencies = [ [[package]] name = "either" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" [[package]] name = "elliptic-curve" @@ -2405,7 +2396,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -2959,6 +2950,17 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "event-listener" +version = "5.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ad6fd685ce13acd6d9541a30f6db6567a7a24c9ffd4ba2955d29e3f22c8b27" +dependencies = [ + "concurrent-queue", + "parking", + "pin-project-lite", +] + [[package]] name = "event-listener-strategy" version = "0.4.0" @@ -2969,6 +2971,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "event-listener-strategy" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "feedafcaa9b749175d5ac357452a9d41ea2911da598fde46ce1fe02c37751291" +dependencies = [ + "event-listener 5.1.0", + "pin-project-lite", +] + [[package]] name = "execution_engine_integration" version = "0.1.0" @@ -3057,9 +3069,9 @@ dependencies = [ [[package]] name = "eyre" -version = "0.6.11" +version = "0.6.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6267a1fa6f59179ea4afc8e50fd8612a3cc60bc858f786ff877a4a8cb042799" +checksum = "7cd915d99f24784cdc19fd37ef22b97e3ff0ae756c7e492e9fbfe897d61e2aec" dependencies = [ "indenter", "once_cell", @@ -3131,9 +3143,9 @@ checksum = "ec54ac60a7f2ee9a97cad9946f9bf629a3bc6a7ae59e68983dc9318f5a54b81a" [[package]] name = "fiat-crypto" -version = "0.2.5" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27573eac26f4dd11e2b1916c3fe1baa56407c83c71a773a8ba17ec0bca03b6b7" +checksum = "1676f435fc1dadde4d03e43f5d62b259e1ce5f40bd4ffb21db2b42ebe59c1382" [[package]] name = "field-offset" @@ -3357,7 +3369,7 @@ checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -3514,7 +3526,7 @@ checksum = "53010ccb100b96a67bc32c0175f0ed1426b31b655d562898e57325f81c023ac0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -3569,7 +3581,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.11", - "indexmap 2.2.1", + "indexmap 2.2.3", "slab", "tokio", "tokio-util 0.7.10", @@ -3588,7 +3600,7 @@ dependencies = [ "futures-sink", "futures-util", "http 1.0.0", - "indexmap 2.2.1", + "indexmap 2.2.3", "slab", "tokio", "tokio-util 0.7.10", @@ -3700,9 +3712,9 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.3.4" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d3d0e0f38255e7fa3cf31335b3a56f05febd18025f4db5ef7a0cfb4f8da651f" +checksum = "bd5256b483761cd23699d0da46cc6fd2ee3be420bbe6d020ae4a091e70b7e9fd" [[package]] name = "hex" @@ -3796,7 +3808,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b" dependencies = [ - "crypto-mac 0.11.0", + "crypto-mac 0.11.1", "digest 0.9.0", ] @@ -3999,7 +4011,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2 0.4.10", + "socket2 0.5.5", "tokio", "tower-service", "tracing", @@ -4054,12 +4066,11 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bdea9aac0dbe5a9240d68cfd9501e2db94222c6dc06843e06640b9e07f0fdc67" +checksum = "ca38ef113da30126bbff9cd1705f9273e15d45498615d138b0c20279ac7a76aa" dependencies = [ "bytes", - "futures-channel", "futures-util", "http 1.0.0", "http-body 1.0.0", @@ -4067,14 +4078,13 @@ dependencies = [ "pin-project-lite", "socket2 0.5.5", "tokio", - "tracing", ] [[package]] name = "iana-time-zone" -version = "0.1.59" +version = "0.1.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6a67363e2aa4443928ce15e57ebae94fd8949958fd1223c4cfc0cd473ad7539" +checksum = "e7ffbb5a1b541ea2561f8c41c087286cc091e21e556a4f09a8f6cbf17b69b141" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -4262,9 +4272,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.2.1" +version = "2.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "433de089bd45971eecf4668ee0ee8f4cec17db4f8bd8f7bc3197a6ce37aa7d9b" +checksum = "233cf39063f058ea2caae4091bf4a3ef70a653afbc026f5c4a4135d114e3c177" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -4315,7 +4325,7 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eae7b9aee968036d54dce06cebaefd919e4472e753296daccd6d344e3e2df0c2" dependencies = [ - "hermit-abi 0.3.4", + "hermit-abi 0.3.6", "libc", "windows-sys 0.48.0", ] @@ -4386,18 +4396,18 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c37f63953c4c63420ed5fd3d6d398c719489b9f872b9fa683262f8edd363c7d" +checksum = "ab46a6e9526ddef3ae7f787c06f0f2600639ba80ea3eade3d8e670a2230f51d6" dependencies = [ "libc", ] [[package]] name = "js-sys" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a1d36f1235bc969acba30b7f5990b864423a6068a10f7c90ae8f0112e3a59d1" +checksum = "406cda4b368d531c842222cf9d2600a9a4acce8d29423695379c6868a143a9ee" dependencies = [ "wasm-bindgen", ] @@ -4577,9 +4587,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.152" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libflate" @@ -4937,7 +4947,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -5525,9 +5535,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7810e0be55b428ada41041c41f32c9f1a42817901b4ccf45fa3d4b6561e74c7" +checksum = "9d811f3e15f28568be3407c8e7fdb6514c1cda3cb30683f15b6a1a1dc4ea14a7" dependencies = [ "adler", ] @@ -5545,9 +5555,9 @@ dependencies = [ [[package]] name = "mock_instant" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c1a54de846c4006b88b1516731cc1f6026eb5dc4bcb186aa071ef66d40524ec" +checksum = "9366861eb2a2c436c20b12c8dbec5f798cea6b47ad99216be0282942e2c81ea0" [[package]] name = "monitoring_api" @@ -5864,21 +5874,26 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" -version = "0.1.45" +version = "0.1.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225d3389fb3509a24c93f5c29eb6bde2586b98d9f016636dff58d7c6f7569cd9" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" dependencies = [ - "autocfg", "num-traits", ] [[package]] name = "num-iter" -version = "0.1.43" +version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d03e6c028c5dc5cac6e2dec0efda81fc887605bb3d884578bb6d6bf7514e252" +checksum = "d869c01cc0c455284163fd0092f1f93835385ccab5a98a0dcc497b2f8bf055a9" dependencies = [ "autocfg", "num-integer", @@ -5887,9 +5902,9 @@ dependencies = [ [[package]] name = "num-traits" -version = "0.2.17" +version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39e3200413f237f41ab11ad6d161bc7239c84dcb631773ccd7de3dfe4b5c267c" +checksum = "da0df0e5185db44f69b44f26786fe401b6c293d1907744beaa7fa62b2e5a517a" dependencies = [ "autocfg", "libm", @@ -5901,15 +5916,15 @@ version = "1.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" dependencies = [ - "hermit-abi 0.3.4", + "hermit-abi 0.3.6", "libc", ] [[package]] name = "num_threads" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2819ce041d2ee131036f4fc9d6ae7ae125a3a40e97ba64d04fe799ad9dabbb44" +checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" dependencies = [ "libc", ] @@ -6005,7 +6020,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -6016,9 +6031,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf" [[package]] name = "openssl-src" -version = "300.2.2+3.2.1" +version = "300.2.3+3.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bbfad0063610ac26ee79f7484739e2b07555a75c42453b89263830b5c8103bc" +checksum = "5cff92b6f71555b61bb9315f7c64da3ca43d87531622120fea0195fc761b4843" dependencies = [ "cc", ] @@ -6206,7 +6221,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d95f5254224e617595d2cc3cc73ff0a5eaf2637519e25f03388154e9378b6ffa" dependencies = [ - "crypto-mac 0.11.0", + "crypto-mac 0.11.1", ] [[package]] @@ -6317,7 +6332,7 @@ checksum = "266c042b60c9c76b8d53061e52b2e0d1116abc57cefc8c5cd671619a56ac3690" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -6365,9 +6380,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.29" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2900ede94e305130c13ddd391e0ab7cbaeb783945ae07a279c268cb05109c6cb" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "platforms" @@ -6427,14 +6442,14 @@ dependencies = [ [[package]] name = "polling" -version = "3.3.2" +version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "545c980a3880efd47b2e262f6a4bb6daad6555cf3367aa9c4e52895f69537a41" +checksum = "24f040dee2588b4963afb4e420540439d126f73fdacf4a9c486a96d840bac3c9" dependencies = [ "cfg-if", "concurrent-queue", "pin-project-lite", - "rustix 0.38.30", + "rustix 0.38.31", "tracing", "windows-sys 0.52.0", ] @@ -6459,7 +6474,7 @@ dependencies = [ "cfg-if", "cpufeatures", "opaque-debug", - "universal-hash 0.4.0", + "universal-hash 0.4.1", ] [[package]] @@ -6539,7 +6554,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" dependencies = [ "proc-macro2", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -6656,7 +6671,7 @@ checksum = "440f724eba9f6996b75d63681b0a92b06947f1457076d503a4d2e2c8f56442b8" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -7016,9 +7031,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "reqwest" -version = "0.11.23" +version = "0.11.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37b1ae8d9ac08420c66222fb9096fc5de435c3c48542bc5336c51892cffafb41" +checksum = "c6920094eb85afde5e4a138be3f2de8bbdf28000f0029e72c45025a56b042251" dependencies = [ "base64 0.21.7", "bytes", @@ -7044,6 +7059,7 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", + "sync_wrapper", "system-configuration", "tokio", "tokio-native-tls", @@ -7107,16 +7123,17 @@ dependencies = [ [[package]] name = "ring" -version = "0.17.7" +version = "0.17.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "688c63d65483050968b2a8937f7995f443e27041a0f7700aa59b0822aedebb74" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" dependencies = [ "cc", + "cfg-if", "getrandom", "libc", "spin 0.9.8", "untrusted 0.9.0", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -7290,9 +7307,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.30" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ "bitflags 2.4.2", "errno", @@ -7308,7 +7325,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9d5a6813c0759e4609cd494e8e725babae6a2ca7b62a5536a13daaec6fcb7ba" dependencies = [ "log", - "ring 0.17.7", + "ring 0.17.8", "rustls-webpki", "sct", ] @@ -7328,7 +7345,7 @@ version = "0.101.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "untrusted 0.9.0", ] @@ -7461,7 +7478,7 @@ version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" dependencies = [ - "ring 0.17.7", + "ring 0.17.8", "untrusted 0.9.0", ] @@ -7594,7 +7611,7 @@ checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -7626,7 +7643,7 @@ checksum = "0b2e6b945e9d3df726b65d6ee24060aff8e3533d431f677a9695db04eff9dfdb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -7678,7 +7695,7 @@ version = "0.9.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adf8a49373e98a4c5f0ceb5d05aa7c648d75f63774981ed95b7c7443bbd50c6e" dependencies = [ - "indexmap 2.2.1", + "indexmap 2.2.3", "itoa", "ryu", "serde", @@ -8043,7 +8060,7 @@ dependencies = [ "chacha20poly1305", "curve25519-dalek", "rand_core", - "ring 0.17.7", + "ring 0.17.8", "rustc_version 0.4.0", "sha2 0.10.8", "subtle", @@ -8235,9 +8252,9 @@ dependencies = [ [[package]] name = "subtle" -version = "2.5.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81cdd64d312baedb58e21336b31bc043b77e01cc99033ce76ef539f78e965ebc" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "superstruct" @@ -8275,9 +8292,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.48" +version = "2.0.49" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f3531638e407dfc0814761abb7c00a5b54992b849452a0646b7f65c9f770f3f" +checksum = "915aea9e586f80826ee59f8453c1101f9d1c4b3964cd2460185ee8e299ada496" dependencies = [ "proc-macro2", "quote", @@ -8390,14 +8407,13 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.9.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" dependencies = [ "cfg-if", "fastrand 2.0.1", - "redox_syscall 0.4.1", - "rustix 0.38.30", + "rustix 0.38.31", "windows-sys 0.52.0", ] @@ -8465,22 +8481,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54378c645627613241d077a3a79db965db602882668f9136ac42af9ecb730ad" +checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.56" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa0faa943b50f3db30a20aa7e265dbc66076993efed8463e8de414e5d06d3471" +checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -8504,13 +8520,14 @@ dependencies = [ [[package]] name = "time" -version = "0.3.31" +version = "0.3.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f657ba42c3f86e7680e53c8cd3af8abbe56b5491790b46e22e19c0d57463583e" +checksum = "c8248b6521bb14bc45b4067159b9b6ad792e2d6d754d6c41fb50e29fefe38749" dependencies = [ "deranged", "itoa", "libc", + "num-conv", "num_threads", "powerfmt", "serde", @@ -8526,10 +8543,11 @@ checksum = "ef927ca75afb808a4d64dd374f00a2adf8d0fcff8e7b184af886c3c87ec4a3f3" [[package]] name = "time-macros" -version = "0.2.16" +version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26197e33420244aeb70c3e8c78376ca46571bc4e701e4791c2cd9f57dcb3a43f" +checksum = "7ba3a3ef41e6672a2f0f001392bb5dcd3ff0a9992d618ca761a11c3121547774" dependencies = [ + "num-conv", "time-core", ] @@ -8599,9 +8617,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.35.1" +version = "1.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c89b4efa943be685f629b149f53829423f8f5531ea21249408e8e2f8671ec104" +checksum = "61285f6515fa018fb2d1e46eb21223fff441ee8db5d0f1435e8ab4f5cdb80931" dependencies = [ "backtrace", "bytes", @@ -8633,7 +8651,7 @@ checksum = "5b8a1e28f2deaa14e508979454cb3a223b10b938b45af148bc0986de36f1923b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -8761,7 +8779,7 @@ version = "0.19.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" dependencies = [ - "indexmap 2.2.1", + "indexmap 2.2.3", "serde", "serde_spanned", "toml_datetime", @@ -8774,7 +8792,7 @@ version = "0.20.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70f427fce4d84c72b5b732388bf4a9f4531b53f74e2887e3ecb2481f68f66d81" dependencies = [ - "indexmap 2.2.1", + "indexmap 2.2.3", "toml_datetime", "winnow", ] @@ -8839,7 +8857,7 @@ checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -9081,9 +9099,9 @@ checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c" [[package]] name = "universal-hash" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8326b2c654932e3e4f9196e69d08fdf7cfd718e1dc6f66b347e6024a0c961402" +checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05" dependencies = [ "generic-array", "subtle", @@ -9400,9 +9418,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasm-bindgen" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1223296a201415c7fad14792dbefaace9bd52b62d33453ade1c5b5f07555406" +checksum = "c1e124130aee3fb58c5bdd6b639a0509486b0338acaaae0c84a5124b0f588b7f" dependencies = [ "cfg-if", "wasm-bindgen-macro", @@ -9410,24 +9428,24 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcdc935b63408d58a32f8cc9738a0bffd8f05cc7c002086c6ef20b7312ad9dcd" +checksum = "c9e7e1900c352b609c8488ad12639a311045f40a35491fb69ba8c12f758af70b" dependencies = [ "bumpalo", "log", "once_cell", "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.40" +version = "0.4.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bde2032aeb86bdfaecc8b261eef3cba735cc426c1f3a3416d1e0791be95fc461" +checksum = "877b9c3f61ceea0e56331985743b13f3d25c406a7098d45180fb5f09bc19ed97" dependencies = [ "cfg-if", "js-sys", @@ -9437,9 +9455,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e4c238561b2d428924c49815533a8b9121c664599558a5d9ec51f8a1740a999" +checksum = "b30af9e2d358182b5c7449424f017eba305ed32a7010509ede96cdc4696c46ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -9447,28 +9465,28 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bae1abb6806dc1ad9e560ed242107c0f6c84335f1749dd4e8ddb012ebd5e25a7" +checksum = "642f325be6301eb8107a83d12a8ac6c1e1c54345a7ef1a9261962dfefda09e66" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", "wasm-bindgen-backend", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.90" +version = "0.2.91" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d91413b1c31d7539ba5ef2451af3f0b833a005eb27a631cec32bc0635a8602b" +checksum = "4f186bd2dcf04330886ce82d6f33dd75a7bfcf69ecf5763b89fcde53b6ac9838" [[package]] name = "wasm-streams" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4609d447824375f43e1ffbc051b50ad8f4b3ae8219680c94452ea05eb240ac7" +checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129" dependencies = [ "futures-util", "js-sys", @@ -9529,9 +9547,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.67" +version = "0.3.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58cd2333b6e0be7a39605f0e255892fd7418a682d8da8fe042fe25128794d2ed" +checksum = "96565907687f7aceb35bc5fc03770a8a0471d82e479f25832f54a0e3f4b28446" dependencies = [ "js-sys", "wasm-bindgen", @@ -9565,9 +9583,9 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.25.3" +version = "0.25.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1778a42e8b3b90bff8d0f5032bf22250792889a5cdc752aa0020c84abe3aaf10" +checksum = "5f20c57d8d7db6d3b86154206ae5d8fba62dd39573114de97c2cb0578251f8e1" [[package]] name = "which" @@ -9578,7 +9596,7 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.30", + "rustix 0.38.31", ] [[package]] @@ -9874,9 +9892,9 @@ checksum = "dff9641d1cd4be8d1a070daf9e3773c5f67e78b4d9d42263020c057706765c04" [[package]] name = "winnow" -version = "0.5.35" +version = "0.5.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1931d78a9c73861da0134f453bb1f790ce49b2e30eba8410b4b79bac72b46a2d" +checksum = "f593a95398737aeed53e489c785df13f3618e41dbcd6718c6addbf1395aa6876" dependencies = [ "memchr", ] @@ -9927,9 +9945,9 @@ dependencies = [ [[package]] name = "x25519-dalek" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb66477291e7e8d2b0ff1bcb900bf29489a9692816d79874bea351e7a8b6de96" +checksum = "c7e468321c81fb07fa7f4c636c3972b9100f0346e5b6a9f2bd0603a52f7ed277" dependencies = [ "curve25519-dalek", "rand_core", @@ -10035,7 +10053,7 @@ checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -10055,7 +10073,7 @@ checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.48", + "syn 2.0.49", ] [[package]] @@ -10064,7 +10082,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" dependencies = [ - "aes 0.8.3", + "aes 0.8.4", "byteorder", "bzip2", "constant_time_eq", diff --git a/beacon_node/lighthouse_network/src/rpc/handler.rs b/beacon_node/lighthouse_network/src/rpc/handler.rs index 03f4761ff08..f4971c18d31 100644 --- a/beacon_node/lighthouse_network/src/rpc/handler.rs +++ b/beacon_node/lighthouse_network/src/rpc/handler.rs @@ -24,7 +24,7 @@ use std::{ task::{Context, Poll}, time::{Duration, Instant}, }; -use tokio::time::{sleep_until, Instant as TInstant, Sleep}; +use tokio::time::{sleep, Sleep}; use tokio_util::time::{delay_queue, DelayQueue}; use types::{EthSpec, ForkContext}; @@ -32,7 +32,7 @@ use types::{EthSpec, ForkContext}; const IO_ERROR_RETRIES: u8 = 3; /// Maximum time given to the handler to perform shutdown operations. -const SHUTDOWN_TIMEOUT_SECS: u8 = 15; +const SHUTDOWN_TIMEOUT_SECS: u64 = 15; /// Maximum number of simultaneous inbound substreams we keep for this peer. const MAX_INBOUND_SUBSTREAMS: usize = 32; @@ -266,9 +266,9 @@ where self.dial_queue.push((id, OutboundRequest::Goodbye(reason))); } - self.state = HandlerState::ShuttingDown(Box::pin(sleep_until( - TInstant::now() + Duration::from_secs(SHUTDOWN_TIMEOUT_SECS as u64), - ))); + self.state = HandlerState::ShuttingDown(Box::pin(sleep(Duration::from_secs( + SHUTDOWN_TIMEOUT_SECS, + )))); } } @@ -349,23 +349,7 @@ where } fn connection_keep_alive(&self) -> bool { - // Check that we don't have outbound items pending for dialing, nor dialing, nor - // established. Also check that there are no established inbound substreams. - // Errors and events need to be reported back, so check those too. - match self.state { - HandlerState::ShuttingDown(_) => { - !self.dial_queue.is_empty() - || !self.outbound_substreams.is_empty() - || !self.inbound_substreams.is_empty() - || !self.events_out.is_empty() - || !self.dial_negotiated != 0 - } - HandlerState::Deactivated => { - // Regardless of events, the timeout has expired. Force the disconnect. - false - } - _ => true, - } + !matches!(self.state, HandlerState::Deactivated) } fn poll( @@ -395,7 +379,7 @@ where match delay.as_mut().poll(cx) { Poll::Ready(_) => { self.state = HandlerState::Deactivated; - debug!(self.log, "Handler deactivated"); + debug!(self.log, "Shutdown timeout elapsed, Handler deactivated"); return Poll::Ready(ConnectionHandlerEvent::NotifyBehaviour( HandlerEvent::Close(RPCError::Disconnected), )); @@ -844,6 +828,8 @@ where && self.events_out.is_empty() && self.dial_negotiated == 0 { + debug!(self.log, "Goodbye sent, Handler deactivated"); + self.state = HandlerState::Deactivated; return Poll::Ready(ConnectionHandlerEvent::NotifyBehaviour( HandlerEvent::Close(RPCError::Disconnected), )); diff --git a/beacon_node/lighthouse_network/src/rpc/mod.rs b/beacon_node/lighthouse_network/src/rpc/mod.rs index 3606438fb99..e22e5273866 100644 --- a/beacon_node/lighthouse_network/src/rpc/mod.rs +++ b/beacon_node/lighthouse_network/src/rpc/mod.rs @@ -223,7 +223,7 @@ where fn handle_established_inbound_connection( &mut self, - _connection_id: ConnectionId, + connection_id: ConnectionId, peer_id: PeerId, _local_addr: &libp2p::Multiaddr, _remote_addr: &libp2p::Multiaddr, @@ -238,9 +238,9 @@ where }, (), ); - // NOTE: this is needed because PeerIds have interior mutability. - let peer_repr = peer_id.to_string(); - let log = self.log.new(slog::o!("peer_id" => peer_repr)); + let log = self + .log + .new(slog::o!("peer_id" => peer_id.to_string(), "connection_id" => connection_id.to_string())); let handler = RPCHandler::new( protocol, self.fork_context.clone(), @@ -253,7 +253,7 @@ where fn handle_established_outbound_connection( &mut self, - _connection_id: ConnectionId, + connection_id: ConnectionId, peer_id: PeerId, _addr: &libp2p::Multiaddr, _role_override: libp2p::core::Endpoint, @@ -269,9 +269,10 @@ where (), ); - // NOTE: this is needed because PeerIds have interior mutability. - let peer_repr = peer_id.to_string(); - let log = self.log.new(slog::o!("peer_id" => peer_repr)); + let log = self + .log + .new(slog::o!("peer_id" => peer_id.to_string(), "connection_id" => connection_id.to_string())); + let handler = RPCHandler::new( protocol, self.fork_context.clone(), From 50c423ad88aa584edf9aca18e5d1d1ee26b9ad1f Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Tue, 20 Feb 2024 15:19:17 +1100 Subject: [PATCH 23/24] Revert libp2p metrics (#4870) (#5265) * Revert "improve libp2p connected peer metrics (#4870)" This reverts commit 0c3fef59b3c6697ee84e4f89a8537c097eacfc5a. --- beacon_node/http_api/src/lib.rs | 10 ++- .../lighthouse_network/src/discovery/mod.rs | 8 +-- beacon_node/lighthouse_network/src/metrics.rs | 68 ++++++++++++++++--- .../src/peer_manager/mod.rs | 18 +++++ .../src/peer_manager/network_behaviour.rs | 26 +++---- common/lighthouse_metrics/src/lib.rs | 10 --- common/system_health/src/lib.rs | 25 ++----- 7 files changed, 100 insertions(+), 65 deletions(-) diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index b39450d7354..a9b245e7987 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -68,7 +68,7 @@ use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; use sysinfo::{System, SystemExt}; -use system_health::{observe_nat, observe_system_health_bn}; +use system_health::observe_system_health_bn; use task_spawner::{Priority, TaskSpawner}; use tokio::sync::{ mpsc::{Sender, UnboundedSender}, @@ -3965,7 +3965,13 @@ pub fn serve( .and(warp::path::end()) .then(|task_spawner: TaskSpawner| { task_spawner.blocking_json_task(Priority::P1, move || { - Ok(api_types::GenericResponse::from(observe_nat())) + Ok(api_types::GenericResponse::from( + lighthouse_network::metrics::NAT_OPEN + .as_ref() + .map(|v| v.get()) + .unwrap_or(0) + != 0, + )) }) }); diff --git a/beacon_node/lighthouse_network/src/discovery/mod.rs b/beacon_node/lighthouse_network/src/discovery/mod.rs index c1781c85832..6659ba1d26f 100644 --- a/beacon_node/lighthouse_network/src/discovery/mod.rs +++ b/beacon_node/lighthouse_network/src/discovery/mod.rs @@ -1003,13 +1003,11 @@ impl NetworkBehaviour for Discovery { } discv5::Event::SocketUpdated(socket_addr) => { info!(self.log, "Address updated"; "ip" => %socket_addr.ip(), "udp_port" => %socket_addr.port()); - // We have SOCKET_UPDATED messages. This occurs when discovery has a majority of - // users reporting an external port and our ENR gets updated. - // Which means we are able to do NAT traversal. - metrics::set_gauge_vec(&metrics::NAT_OPEN, &["discv5"], 1); - + metrics::inc_counter(&metrics::ADDRESS_UPDATE_COUNT); + metrics::check_nat(); // Discv5 will have updated our local ENR. We save the updated version // to disk. + if (self.update_ports.tcp4 && socket_addr.is_ipv4()) || (self.update_ports.tcp6 && socket_addr.is_ipv6()) { diff --git a/beacon_node/lighthouse_network/src/metrics.rs b/beacon_node/lighthouse_network/src/metrics.rs index ada48d6ebd0..ae02b689d81 100644 --- a/beacon_node/lighthouse_network/src/metrics.rs +++ b/beacon_node/lighthouse_network/src/metrics.rs @@ -1,17 +1,29 @@ pub use lighthouse_metrics::*; lazy_static! { - pub static ref NAT_OPEN: Result = try_create_int_gauge_vec( + pub static ref NAT_OPEN: Result = try_create_int_counter( "nat_open", - "An estimate indicating if the local node is reachable from external nodes", - &["protocol"] + "An estimate indicating if the local node is exposed to the internet." ); pub static ref ADDRESS_UPDATE_COUNT: Result = try_create_int_counter( "libp2p_address_update_total", "Count of libp2p socked updated events (when our view of our IP address has changed)" ); - pub static ref PEERS_CONNECTED: Result = - try_create_int_gauge_vec("libp2p_peers", "Count of libp2p peers currently connected", &["direction", "transport"]); + pub static ref PEERS_CONNECTED: Result = try_create_int_gauge( + "libp2p_peers", + "Count of libp2p peers currently connected" + ); + + pub static ref TCP_PEERS_CONNECTED: Result = try_create_int_gauge( + "libp2p_tcp_peers", + "Count of libp2p peers currently connected via TCP" + ); + + pub static ref QUIC_PEERS_CONNECTED: Result = try_create_int_gauge( + "libp2p_quic_peers", + "Count of libp2p peers currently connected via QUIC" + ); + pub static ref PEER_CONNECT_EVENT_COUNT: Result = try_create_int_counter( "libp2p_peer_connect_event_total", "Count of libp2p peer connect events (not the current number of connected peers)" @@ -20,10 +32,13 @@ lazy_static! { "libp2p_peer_disconnect_event_total", "Count of libp2p peer disconnect events" ); - pub static ref DISCOVERY_BYTES: Result = try_create_int_gauge_vec( - "discovery_bytes", - "The number of bytes sent and received in discovery", - &["direction"] + pub static ref DISCOVERY_SENT_BYTES: Result = try_create_int_gauge( + "discovery_sent_bytes", + "The number of bytes sent in discovery" + ); + pub static ref DISCOVERY_RECV_BYTES: Result = try_create_int_gauge( + "discovery_recv_bytes", + "The number of bytes received in discovery" ); pub static ref DISCOVERY_QUEUE: Result = try_create_int_gauge( "discovery_queue_size", @@ -120,6 +135,17 @@ lazy_static! { &["type"] ); + /* + * Inbound/Outbound peers + */ + /// The number of peers that dialed us. + pub static ref NETWORK_INBOUND_PEERS: Result = + try_create_int_gauge("network_inbound_peers","The number of peers that are currently connected that have dialed us."); + + /// The number of peers that we dialed us. + pub static ref NETWORK_OUTBOUND_PEERS: Result = + try_create_int_gauge("network_outbound_peers","The number of peers that are currently connected that we dialed."); + /* * Peer Reporting */ @@ -130,11 +156,31 @@ lazy_static! { ); } +/// Checks if we consider the NAT open. +/// +/// Conditions for an open NAT: +/// 1. We have 1 or more SOCKET_UPDATED messages. This occurs when discovery has a majority of +/// users reporting an external port and our ENR gets updated. +/// 2. We have 0 SOCKET_UPDATED messages (can be true if the port was correct on boot), then we +/// rely on whether we have any inbound messages. If we have no socket update messages, but +/// manage to get at least one inbound peer, we are exposed correctly. +pub fn check_nat() { + // NAT is already deemed open. + if NAT_OPEN.as_ref().map(|v| v.get()).unwrap_or(0) != 0 { + return; + } + if ADDRESS_UPDATE_COUNT.as_ref().map(|v| v.get()).unwrap_or(0) != 0 + || NETWORK_INBOUND_PEERS.as_ref().map(|v| v.get()).unwrap_or(0) != 0_i64 + { + inc_counter(&NAT_OPEN); + } +} + pub fn scrape_discovery_metrics() { let metrics = discv5::metrics::Metrics::from(discv5::Discv5::::raw_metrics()); set_float_gauge(&DISCOVERY_REQS, metrics.unsolicited_requests_per_second); set_gauge(&DISCOVERY_SESSIONS, metrics.active_sessions as i64); - set_gauge_vec(&DISCOVERY_BYTES, &["inbound"], metrics.bytes_recv as i64); - set_gauge_vec(&DISCOVERY_BYTES, &["outbound"], metrics.bytes_sent as i64); + set_gauge(&DISCOVERY_SENT_BYTES, metrics.bytes_sent as i64); + set_gauge(&DISCOVERY_RECV_BYTES, metrics.bytes_recv as i64); } diff --git a/beacon_node/lighthouse_network/src/peer_manager/mod.rs b/beacon_node/lighthouse_network/src/peer_manager/mod.rs index 8227c3420b2..e4976a0d374 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/mod.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/mod.rs @@ -726,14 +726,29 @@ impl PeerManager { return; } + let mut connected_peer_count = 0; + let mut inbound_connected_peers = 0; + let mut outbound_connected_peers = 0; let mut clients_per_peer = HashMap::new(); for (_peer, peer_info) in self.network_globals.peers.read().connected_peers() { + connected_peer_count += 1; + if let PeerConnectionStatus::Connected { n_in, .. } = peer_info.connection_status() { + if *n_in > 0 { + inbound_connected_peers += 1; + } else { + outbound_connected_peers += 1; + } + } *clients_per_peer .entry(peer_info.client().kind.to_string()) .or_default() += 1; } + metrics::set_gauge(&metrics::PEERS_CONNECTED, connected_peer_count); + metrics::set_gauge(&metrics::NETWORK_INBOUND_PEERS, inbound_connected_peers); + metrics::set_gauge(&metrics::NETWORK_OUTBOUND_PEERS, outbound_connected_peers); + for client_kind in ClientKind::iter() { let value = clients_per_peer.get(&client_kind.to_string()).unwrap_or(&0); metrics::set_gauge_vec( @@ -838,8 +853,11 @@ impl PeerManager { // start a ping and status timer for the peer self.status_peers.insert(*peer_id); + let connected_peers = self.network_globals.connected_peers() as i64; + // increment prometheus metrics metrics::inc_counter(&metrics::PEER_CONNECT_EVENT_COUNT); + metrics::set_gauge(&metrics::PEERS_CONNECTED, connected_peers); true } diff --git a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs index 1c9274af233..cb60906f632 100644 --- a/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs +++ b/beacon_node/lighthouse_network/src/peer_manager/network_behaviour.rs @@ -154,8 +154,6 @@ impl NetworkBehaviour for PeerManager { self.on_dial_failure(peer_id); } FromSwarm::ExternalAddrConfirmed(_) => { - // We have an external address confirmed, means we are able to do NAT traversal. - metrics::set_gauge_vec(&metrics::NAT_OPEN, &["libp2p"], 1); // TODO: we likely want to check this against our assumed external tcp // address } @@ -245,14 +243,14 @@ impl PeerManager { self.events.push(PeerManagerEvent::MetaData(peer_id)); } + // Check NAT if metrics are enabled + if self.network_globals.local_enr.read().udp4().is_some() { + metrics::check_nat(); + } + // increment prometheus metrics if self.metrics_enabled { let remote_addr = endpoint.get_remote_address(); - let direction = if endpoint.is_dialer() { - "outbound" - } else { - "inbound" - }; match remote_addr.iter().find(|proto| { matches!( proto, @@ -260,10 +258,10 @@ impl PeerManager { ) }) { Some(multiaddr::Protocol::QuicV1) => { - metrics::inc_gauge_vec(&metrics::PEERS_CONNECTED, &[direction, "quic"]); + metrics::inc_gauge(&metrics::QUIC_PEERS_CONNECTED); } Some(multiaddr::Protocol::Tcp(_)) => { - metrics::inc_gauge_vec(&metrics::PEERS_CONNECTED, &[direction, "tcp"]); + metrics::inc_gauge(&metrics::TCP_PEERS_CONNECTED); } Some(_) => unreachable!(), None => { @@ -341,12 +339,6 @@ impl PeerManager { let remote_addr = endpoint.get_remote_address(); // Update the prometheus metrics if self.metrics_enabled { - let direction = if endpoint.is_dialer() { - "outbound" - } else { - "inbound" - }; - match remote_addr.iter().find(|proto| { matches!( proto, @@ -354,10 +346,10 @@ impl PeerManager { ) }) { Some(multiaddr::Protocol::QuicV1) => { - metrics::dec_gauge_vec(&metrics::PEERS_CONNECTED, &[direction, "quic"]); + metrics::dec_gauge(&metrics::QUIC_PEERS_CONNECTED); } Some(multiaddr::Protocol::Tcp(_)) => { - metrics::dec_gauge_vec(&metrics::PEERS_CONNECTED, &[direction, "tcp"]); + metrics::dec_gauge(&metrics::TCP_PEERS_CONNECTED); } // If it's an unknown protocol we already logged when connection was established. _ => {} diff --git a/common/lighthouse_metrics/src/lib.rs b/common/lighthouse_metrics/src/lib.rs index ba9900cc507..5d25bb313f6 100644 --- a/common/lighthouse_metrics/src/lib.rs +++ b/common/lighthouse_metrics/src/lib.rs @@ -244,16 +244,6 @@ pub fn inc_counter_vec(int_counter_vec: &Result, name: &[&str]) { } } -/// Sets the `int_counter_vec` with the given `name` to the `amount`, -/// should only be called with `ammount`s equal or above the current value -/// as per Prometheus spec, the `counter` type should only go up. -pub fn set_counter_vec_by(int_counter_vec: &Result, name: &[&str], amount: u64) { - if let Some(counter) = get_int_counter(int_counter_vec, name) { - counter.reset(); - counter.inc_by(amount); - } -} - pub fn inc_counter_vec_by(int_counter_vec: &Result, name: &[&str], amount: u64) { if let Some(counter) = get_int_counter(int_counter_vec, name) { counter.inc_by(amount); diff --git a/common/system_health/src/lib.rs b/common/system_health/src/lib.rs index ec64ce31ad3..d10540e506c 100644 --- a/common/system_health/src/lib.rs +++ b/common/system_health/src/lib.rs @@ -198,25 +198,6 @@ pub fn observe_system_health_vc( } } -/// Observes if NAT traversal is possible. -pub fn observe_nat() -> bool { - let discv5_nat = lighthouse_network::metrics::get_int_gauge( - &lighthouse_network::metrics::NAT_OPEN, - &["discv5"], - ) - .map(|g| g.get() == 1) - .unwrap_or_default(); - - let libp2p_nat = lighthouse_network::metrics::get_int_gauge( - &lighthouse_network::metrics::NAT_OPEN, - &["libp2p"], - ) - .map(|g| g.get() == 1) - .unwrap_or_default(); - - discv5_nat && libp2p_nat -} - /// Observes the Beacon Node system health. pub fn observe_system_health_bn( sysinfo: Arc>, @@ -242,7 +223,11 @@ pub fn observe_system_health_bn( .unwrap_or_else(|| (String::from("None"), 0, 0)); // Determine if the NAT is open or not. - let nat_open = observe_nat(); + let nat_open = lighthouse_network::metrics::NAT_OPEN + .as_ref() + .map(|v| v.get()) + .unwrap_or(0) + != 0; SystemHealthBN { system_health, From b5bae6e7a2f52310617a3b93ab7579f3dc9d7880 Mon Sep 17 00:00:00 2001 From: Paul Hauner Date: Wed, 21 Feb 2024 09:12:52 +1100 Subject: [PATCH 24/24] Release v5.0.0 (#5254) * Bump versions --- Cargo.lock | 8 ++++---- beacon_node/Cargo.toml | 2 +- boot_node/Cargo.toml | 2 +- common/lighthouse_version/src/lib.rs | 4 ++-- lcli/Cargo.toml | 2 +- lighthouse/Cargo.toml | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 46fd981aa77..aa17e2f4ceb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -941,7 +941,7 @@ dependencies = [ [[package]] name = "beacon_node" -version = "4.6.0" +version = "5.0.0" dependencies = [ "beacon_chain", "clap", @@ -1194,7 +1194,7 @@ dependencies = [ [[package]] name = "boot_node" -version = "4.6.0" +version = "5.0.0" dependencies = [ "beacon_node", "clap", @@ -4524,7 +4524,7 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "lcli" -version = "4.6.0" +version = "5.0.0" dependencies = [ "account_utils", "beacon_chain", @@ -5100,7 +5100,7 @@ dependencies = [ [[package]] name = "lighthouse" -version = "4.6.0" +version = "5.0.0" dependencies = [ "account_manager", "account_utils", diff --git a/beacon_node/Cargo.toml b/beacon_node/Cargo.toml index 8428a30a3b1..f960251e7a3 100644 --- a/beacon_node/Cargo.toml +++ b/beacon_node/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "beacon_node" -version = "4.6.0" +version = "5.0.0" authors = [ "Paul Hauner ", "Age Manning "] edition = { workspace = true } diff --git a/common/lighthouse_version/src/lib.rs b/common/lighthouse_version/src/lib.rs index af11723487f..10759f94306 100644 --- a/common/lighthouse_version/src/lib.rs +++ b/common/lighthouse_version/src/lib.rs @@ -17,8 +17,8 @@ pub const VERSION: &str = git_version!( // NOTE: using --match instead of --exclude for compatibility with old Git "--match=thiswillnevermatchlol" ], - prefix = "Lighthouse/v4.6.0-", - fallback = "Lighthouse/v4.6.0" + prefix = "Lighthouse/v5.0.0-", + fallback = "Lighthouse/v5.0.0" ); /// Returns `VERSION`, but with platform information appended to the end. diff --git a/lcli/Cargo.toml b/lcli/Cargo.toml index 1bba8242c54..3acf3909b3b 100644 --- a/lcli/Cargo.toml +++ b/lcli/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "lcli" description = "Lighthouse CLI (modeled after zcli)" -version = "4.6.0" +version = "5.0.0" authors = ["Paul Hauner "] edition = { workspace = true } diff --git a/lighthouse/Cargo.toml b/lighthouse/Cargo.toml index b14227e7edf..ffa4727d7f2 100644 --- a/lighthouse/Cargo.toml +++ b/lighthouse/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lighthouse" -version = "4.6.0" +version = "5.0.0" authors = ["Sigma Prime "] edition = { workspace = true } autotests = false