From 3a7a0479befd9242f2a62d4c7fc61dac8457822c Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Mon, 29 May 2023 16:26:51 +1000 Subject: [PATCH 1/3] Add Deneb `BuilderBid` types --- beacon_node/execution_layer/src/lib.rs | 127 +++++++++++++------------ consensus/types/src/builder_bid.rs | 74 ++++++++------ 2 files changed, 110 insertions(+), 91 deletions(-) diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 0e1fddfad16..cfa8516ff5d 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -43,6 +43,7 @@ use tokio_stream::wrappers::WatchStream; use tree_hash::TreeHash; use types::beacon_block_body::KzgCommitments; use types::blob_sidecar::Blobs; +use types::builder_bid::BuilderBid; use types::consts::deneb::BLOB_TX_TYPE; use types::transaction::{AccessTuple, BlobTransaction, EcdsaSignature, SignedBlobTransaction}; use types::{AbstractExecPayload, BeaconStateError, ExecPayload, VersionedHash}; @@ -95,6 +96,31 @@ pub enum ProvenancedPayload

{ Builder(P), } +impl> From> + for ProvenancedPayload> +{ + fn from(value: BuilderBid) -> Self { + let block_proposal_contents = match value { + BuilderBid::Merge(builder_bid) => BlockProposalContents::Payload { + payload: builder_bid.header, + block_value: builder_bid.value, + }, + BuilderBid::Capella(builder_bid) => BlockProposalContents::Payload { + payload: builder_bid.header, + block_value: builder_bid.value, + }, + BuilderBid::Deneb(builder_bid) => BlockProposalContents::PayloadAndBlindedBlobs { + payload: builder_bid.header, + block_value: builder_bid.value, + kzg_commitments: builder_bid.blinded_blobs_bundle.commitments, + blob_roots: builder_bid.blinded_blobs_bundle.blob_roots, + proofs: builder_bid.blinded_blobs_bundle.proofs, + }, + }; + ProvenancedPayload::Builder(block_proposal_contents) + } +} + #[derive(Debug)] pub enum Error { NoEngine, @@ -143,6 +169,13 @@ pub enum BlockProposalContents> { blobs: Blobs, proofs: KzgProofs, }, + PayloadAndBlindedBlobs { + payload: Payload, + block_value: Uint256, + kzg_commitments: KzgCommitments, + blob_roots: VariableList, + proofs: KzgProofs, + }, } impl> From> @@ -188,52 +221,35 @@ impl> BlockProposalContents (payload, Some(kzg_commitments), Some(blobs), Some(proofs)), + Self::PayloadAndBlindedBlobs { + payload, + block_value: _, + kzg_commitments, + blob_roots: _, + proofs, + } => unimplemented!("need to work out how to handle blinded blobs"), } } pub fn payload(&self) -> &Payload { match self { - Self::Payload { - payload, - block_value: _, - } => payload, - Self::PayloadAndBlobs { - payload, - block_value: _, - kzg_commitments: _, - blobs: _, - proofs: _, - } => payload, + Self::Payload { payload, .. } => payload, + Self::PayloadAndBlobs { payload, .. } => payload, + Self::PayloadAndBlindedBlobs { payload, .. } => payload, } } pub fn to_payload(self) -> Payload { match self { - Self::Payload { - payload, - block_value: _, - } => payload, - Self::PayloadAndBlobs { - payload, - block_value: _, - kzg_commitments: _, - blobs: _, - proofs: _, - } => payload, + Self::Payload { payload, .. } => payload, + Self::PayloadAndBlobs { payload, .. } => payload, + Self::PayloadAndBlindedBlobs { payload, .. } => payload, } } pub fn block_value(&self) -> &Uint256 { match self { - Self::Payload { - payload: _, - block_value, - } => block_value, - Self::PayloadAndBlobs { - payload: _, - block_value, - kzg_commitments: _, - blobs: _, - proofs: _, - } => block_value, + Self::Payload { block_value, .. } => block_value, + Self::PayloadAndBlobs { block_value, .. } => block_value, + Self::PayloadAndBlindedBlobs { block_value, .. } => block_value, } } pub fn default_at_fork(fork_name: ForkName) -> Result { @@ -848,7 +864,7 @@ impl ExecutionLayer { self.log(), "Requested blinded execution payload"; "relay_fee_recipient" => match &relay_result { - Ok(Some(r)) => format!("{:?}", r.data.message.header.fee_recipient()), + Ok(Some(r)) => format!("{:?}", r.data.message.header().fee_recipient()), Ok(None) => "empty response".to_string(), Err(_) => "request failed".to_string(), }, @@ -884,7 +900,7 @@ impl ExecutionLayer { Ok(ProvenancedPayload::Local(local)) } (Ok(Some(relay)), Ok(local)) => { - let header = &relay.data.message.header; + let header = &relay.data.message.header(); info!( self.log(), @@ -894,10 +910,10 @@ impl ExecutionLayer { "parent_hash" => ?parent_hash, ); - let relay_value = relay.data.message.value; + let relay_value = relay.data.message.value(); let local_value = *local.block_value(); if !self.inner.always_prefer_builder_payload - && local_value >= relay_value + && local_value >= *relay_value { info!( self.log(), @@ -917,12 +933,7 @@ impl ExecutionLayer { current_fork, spec, ) { - Ok(()) => Ok(ProvenancedPayload::Builder( - BlockProposalContents::Payload { - payload: relay.data.message.header, - block_value: relay.data.message.value, - }, - )), + Ok(()) => Ok(ProvenancedPayload::from(relay.data.message)), Err(reason) if !reason.payload_invalid() => { info!( self.log(), @@ -952,7 +963,7 @@ impl ExecutionLayer { } } (Ok(Some(relay)), Err(local_error)) => { - let header = &relay.data.message.header; + let header = &relay.data.message.header(); info!( self.log(), @@ -971,20 +982,12 @@ impl ExecutionLayer { current_fork, spec, ) { - Ok(()) => Ok(ProvenancedPayload::Builder( - BlockProposalContents::Payload { - payload: relay.data.message.header, - block_value: relay.data.message.value, - }, - )), + Ok(()) => Ok(ProvenancedPayload::from(relay.data.message)), // If the payload is valid then use it. The local EE failed // to produce a payload so we have no alternative. - Err(e) if !e.payload_invalid() => Ok(ProvenancedPayload::Builder( - BlockProposalContents::Payload { - payload: relay.data.message.header, - block_value: relay.data.message.value, - }, - )), + Err(e) if !e.payload_invalid() => { + Ok(ProvenancedPayload::from(relay.data.message)) + } Err(reason) => { metrics::inc_counter_vec( &metrics::EXECUTION_LAYER_GET_PAYLOAD_BUILDER_REJECTIONS, @@ -2046,11 +2049,11 @@ fn verify_builder_bid>( spec: &ChainSpec, ) -> Result<(), Box> { let is_signature_valid = bid.data.verify_signature(spec); - let header = &bid.data.message.header; - let payload_value = bid.data.message.value; + let header = &bid.data.message.header(); + let payload_value = bid.data.message.value(); // Avoid logging values that we can't represent with our Prometheus library. - let payload_value_gwei = bid.data.message.value / 1_000_000_000; + let payload_value_gwei = bid.data.message.value() / 1_000_000_000; if payload_value_gwei <= Uint256::from(i64::max_value()) { metrics::set_gauge_vec( &metrics::EXECUTION_LAYER_PAYLOAD_BIDS, @@ -2066,10 +2069,10 @@ fn verify_builder_bid>( .map(|withdrawals| Withdrawals::::from(withdrawals).tree_hash_root()); let payload_withdrawals_root = header.withdrawals_root().ok(); - if payload_value < profit_threshold { + if *payload_value < profit_threshold { Err(Box::new(InvalidBuilderPayload::LowValue { profit_threshold, - payload_value, + payload_value: *payload_value, })) } else if header.parent_hash() != parent_hash { Err(Box::new(InvalidBuilderPayload::ParentHash { @@ -2099,7 +2102,7 @@ fn verify_builder_bid>( } else if !is_signature_valid { Err(Box::new(InvalidBuilderPayload::Signature { signature: bid.data.signature.clone(), - pubkey: bid.data.message.pubkey, + pubkey: *bid.data.message.pubkey(), })) } else if payload_withdrawals_root != expected_withdrawals_root { Err(Box::new(InvalidBuilderPayload::WithdrawalsRoot { diff --git a/consensus/types/src/builder_bid.rs b/consensus/types/src/builder_bid.rs index e922e81c706..82ce9a69e8d 100644 --- a/consensus/types/src/builder_bid.rs +++ b/consensus/types/src/builder_bid.rs @@ -1,21 +1,46 @@ +use crate::beacon_block_body::KzgCommitments; use crate::{ AbstractExecPayload, ChainSpec, EthSpec, ExecPayload, ExecutionPayloadHeader, ForkName, - ForkVersionDeserialize, SignedRoot, Uint256, + ForkVersionDeserialize, Hash256, KzgProofs, SignedRoot, Uint256, }; use bls::PublicKeyBytes; use bls::Signature; use serde::{Deserialize as De, Deserializer, Serialize as Ser, Serializer}; use serde_derive::{Deserialize, Serialize}; -use serde_with::{serde_as, DeserializeAs, SerializeAs}; +use serde_with::As; +use serde_with::{DeserializeAs, SerializeAs}; +use ssz_types::VariableList; use std::marker::PhantomData; +use superstruct::superstruct; use tree_hash_derive::TreeHash; -#[serde_as] #[derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone)] -#[serde(bound = "E: EthSpec, Payload: ExecPayload")] +#[serde(bound = "E: EthSpec")] +pub struct BlindedBlobsBundle { + pub commitments: KzgCommitments, + pub proofs: KzgProofs, + pub blob_roots: VariableList, +} + +#[superstruct( + variants(Merge, Capella, Deneb), + variant_attributes( + derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone), + serde(bound = "E: EthSpec, Payload: ExecPayload", deny_unknown_fields) + ) +)] +#[derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone)] +#[serde( + bound = "E: EthSpec, Payload: ExecPayload", + deny_unknown_fields, + untagged +)] +#[tree_hash(enum_behaviour = "transparent")] pub struct BuilderBid> { - #[serde_as(as = "BlindedPayloadAsHeader")] + #[serde(with = "As::>")] pub header: Payload, + #[superstruct(only(Deneb))] + pub blinded_blobs_bundle: BlindedBlobsBundle, #[serde(with = "eth2_serde_utils::quoted_u256")] pub value: Uint256, pub pubkey: PublicKeyBytes, @@ -37,32 +62,23 @@ pub struct SignedBuilderBid> { impl> ForkVersionDeserialize for BuilderBid { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + fn deserialize_by_fork<'de, D: Deserializer<'de>>( value: serde_json::value::Value, fork_name: ForkName, ) -> Result { - let convert_err = |_| { - serde::de::Error::custom( - "BuilderBid failed to deserialize: unable to convert payload header to payload", - ) - }; - - #[derive(Deserialize)] - struct Helper { - header: serde_json::Value, - #[serde(with = "eth2_serde_utils::quoted_u256")] - value: Uint256, - pubkey: PublicKeyBytes, - } - let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; - let payload_header = - ExecutionPayloadHeader::deserialize_by_fork::<'de, D>(helper.header, fork_name)?; + let convert_err = + |e| serde::de::Error::custom(format!("BuilderBid failed to deserialize: {:?}", e)); - Ok(Self { - header: Payload::try_from(payload_header).map_err(convert_err)?, - value: helper.value, - pubkey: helper.pubkey, - _phantom_data: Default::default(), + Ok(match fork_name { + ForkName::Merge => Self::Merge(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Capella => Self::Capella(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Deneb => Self::Deneb(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Base | ForkName::Altair => { + return Err(serde::de::Error::custom(format!( + "BuilderBid failed to deserialize: unsupported fork '{}'", + fork_name + ))); + } }) } } @@ -70,7 +86,7 @@ impl> ForkVersionDeserialize impl> ForkVersionDeserialize for SignedBuilderBid { - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + fn deserialize_by_fork<'de, D: Deserializer<'de>>( value: serde_json::value::Value, fork_name: ForkName, ) -> Result { @@ -115,7 +131,7 @@ impl<'de, E: EthSpec, Payload: AbstractExecPayload> DeserializeAs<'de, Payloa impl> SignedBuilderBid { pub fn verify_signature(&self, spec: &ChainSpec) -> bool { self.message - .pubkey + .pubkey() .decompress() .map(|pubkey| { let domain = spec.get_builder_domain(); From 75b6ebe11716692fba8021986e43452c36c0a464 Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 21 Jun 2023 09:52:06 +1000 Subject: [PATCH 2/3] Add blinded flow types --- beacon_node/http_api/tests/tests.rs | 27 ++++++++- common/eth2/src/lib.rs | 4 +- common/eth2/src/types.rs | 90 +++++++++++++++++++++++++++++ consensus/types/src/blob_sidecar.rs | 34 ++++++++++- consensus/types/src/lib.rs | 1 + consensus/types/src/signed_blob.rs | 24 ++++++++ 6 files changed, 174 insertions(+), 6 deletions(-) diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 491c55845d4..d97c9a1a9af 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -2521,7 +2521,8 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data; + .data + .block(); let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); @@ -2544,7 +2545,7 @@ impl ApiTester { for _ in 0..E::slots_per_epoch() { let slot = self.chain.slot().unwrap(); - let block = self + let block_contents = self .client .get_validator_blinded_blocks_modular::( slot, @@ -2555,7 +2556,7 @@ impl ApiTester { .await .unwrap() .data; - assert_eq!(block.slot(), slot); + assert_eq!(block_contents.block().slot(), slot); self.chain.slot_clock.set_slot(slot.as_u64() + 1); } @@ -3028,6 +3029,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3068,6 +3070,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3111,6 +3114,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3160,6 +3164,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3208,6 +3213,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3255,6 +3261,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3301,6 +3308,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3337,6 +3345,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3374,6 +3383,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3417,6 +3427,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3446,6 +3457,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3495,6 +3507,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3534,6 +3547,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3577,6 +3591,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3618,6 +3633,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3655,6 +3671,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3692,6 +3709,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3729,6 +3747,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3779,6 +3798,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3821,6 +3841,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 83cdeddd80d..0fa3c7ce713 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -1432,7 +1432,7 @@ impl BeaconNodeHttpClient { slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, - ) -> Result>, Error> { + ) -> Result>, Error> { self.get_validator_blinded_blocks_modular( slot, randao_reveal, @@ -1452,7 +1452,7 @@ impl BeaconNodeHttpClient { randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, skip_randao_verification: SkipRandaoVerification, - ) -> Result>, Error> { + ) -> Result>, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index bee9b6f1398..834886ce9cc 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -11,6 +11,7 @@ use std::convert::TryFrom; use std::fmt; use std::str::{from_utf8, FromStr}; use std::time::Duration; +use types::blob_sidecar::BlindedBlobSidecarList; pub use types::*; #[cfg(feature = "lighthouse")] @@ -1434,3 +1435,92 @@ impl> ForkVersionDeserialize }) } } + +#[derive(Debug, Clone, Serialize, Deserialize, Encode)] +#[serde(bound = "T: EthSpec")] +pub struct BlindedBeaconBlockAndBlobSidecars> { + pub blinded_block: BeaconBlock, + pub blinded_blob_sidecars: BlindedBlobSidecarList, +} + +impl> ForkVersionDeserialize + for BlindedBeaconBlockAndBlobSidecars +{ + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::value::Value, + fork_name: ForkName, + ) -> Result { + #[derive(Deserialize)] + #[serde(bound = "T: EthSpec")] + struct Helper { + blinded_block: serde_json::Value, + blinded_blob_sidecars: BlindedBlobSidecarList, + } + let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; + + Ok(Self { + blinded_block: BeaconBlock::deserialize_by_fork::<'de, D>( + helper.blinded_block, + fork_name, + )?, + blinded_blob_sidecars: helper.blinded_blob_sidecars, + }) + } +} + +/// A wrapper over a [`BlindedBeaconBlock`] or a [`BlindedBeaconBlockAndBlobSidecars`]. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(untagged)] +#[serde(bound = "T: EthSpec")] +pub enum BlindedBlockContents = BlindedPayload> { + BlockAndBlobSidecars(BlindedBeaconBlockAndBlobSidecars), + Block(BeaconBlock), +} + +impl> BlindedBlockContents { + pub fn block(&self) -> &BeaconBlock { + match self { + BlindedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => { + &block_and_sidecars.blinded_block + } + BlindedBlockContents::Block(block) => block, + } + } +} + +impl> ForkVersionDeserialize + for BlindedBlockContents +{ + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::value::Value, + fork_name: ForkName, + ) -> Result { + match fork_name { + ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + Ok(BlindedBlockContents::Block( + BeaconBlock::deserialize_by_fork::<'de, D>(value, fork_name)?, + )) + } + ForkName::Deneb => Ok(BlindedBlockContents::BlockAndBlobSidecars( + BlindedBeaconBlockAndBlobSidecars::deserialize_by_fork::<'de, D>(value, fork_name)?, + )), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, Encode)] +#[serde(bound = "T: EthSpec")] +pub struct SignedBlindedBeaconBlockAndBlobSidecars> { + pub signed_blinded_block: SignedBeaconBlock, + pub signed_blinded_blob_sidecars: SignedBlindedBlobSidecarList, +} + +/// A wrapper over a [`SignedBlindedBeaconBlock`] or a [`SignedBlindedBeaconBlockAndBlobSidecars`]. +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(untagged)] +#[serde(bound = "T: EthSpec")] +pub enum SignedBlindedBlockContents = BlindedPayload> +{ + BlockAndBlobSidecars(SignedBlindedBeaconBlockAndBlobSidecars), + Block(SignedBeaconBlock), +} diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 2d25de4032b..712b69a0b85 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -1,5 +1,7 @@ use crate::test_utils::TestRandom; -use crate::{Blob, ChainSpec, Domain, EthSpec, Fork, Hash256, SignedBlobSidecar, SignedRoot, Slot}; +use crate::{ + Blob, BlobRoot, ChainSpec, Domain, EthSpec, Fork, Hash256, SignedBlobSidecar, SignedRoot, Slot, +}; use bls::SecretKey; use derivative::Derivative; use kzg::{KzgCommitment, KzgProof}; @@ -123,3 +125,33 @@ impl BlobSidecar { } } } + +#[derive( + Debug, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + Default, + TestRandom, + Derivative, + arbitrary::Arbitrary, +)] +#[derivative(PartialEq, Eq, Hash)] +pub struct BlindedBlobSidecar { + pub block_root: Hash256, + #[serde(with = "serde_utils::quoted_u64")] + pub index: u64, + pub slot: Slot, + pub block_parent_root: Hash256, + #[serde(with = "serde_utils::quoted_u64")] + pub proposer_index: u64, + pub blob_root: BlobRoot, + pub kzg_commitment: KzgCommitment, + pub kzg_proof: KzgProof, +} + +pub type BlindedBlobSidecarList = + VariableList, ::MaxBlobsPerBlock>; diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 46c5c2a4ce8..162a300fde5 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -204,6 +204,7 @@ pub type Address = H160; pub type ForkVersion = [u8; 4]; pub type BLSFieldElement = Uint256; pub type Blob = FixedVector::BytesPerBlob>; +pub type BlobRoot = Hash256; pub type KzgProofs = VariableList::MaxBlobsPerBlock>; pub type VersionedHash = Hash256; pub type Hash64 = ethereum_types::H64; diff --git a/consensus/types/src/signed_blob.rs b/consensus/types/src/signed_blob.rs index aaab02ca783..a85ba93b40a 100644 --- a/consensus/types/src/signed_blob.rs +++ b/consensus/types/src/signed_blob.rs @@ -69,3 +69,27 @@ impl SignedBlobSidecar { self.signature.verify(pubkey, message) } } + +#[derive( + Debug, + Clone, + PartialEq, + Serialize, + Deserialize, + Encode, + Decode, + TestRandom, + TreeHash, + Derivative, + arbitrary::Arbitrary, +)] +#[serde(bound = "T: EthSpec")] +#[arbitrary(bound = "T: EthSpec")] +#[derivative(Hash(bound = "T: EthSpec"))] +pub struct SignedBlindedBlobSidecar { + pub message: Arc>, + pub signature: Signature, +} + +pub type SignedBlindedBlobSidecarList = + VariableList, ::MaxBlobsPerBlock>; From 2f6ee56cb08a6fc673c1e2b3ad0ccb4f9166265a Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Wed, 21 Jun 2023 11:12:49 +1000 Subject: [PATCH 3/3] Implement `GET validator/blinded_blocks/{slot}` --- beacon_node/beacon_chain/src/beacon_chain.rs | 181 +++++++++++------- beacon_node/beacon_chain/src/blob_cache.rs | 18 +- beacon_node/beacon_chain/src/builder.rs | 1 + beacon_node/beacon_chain/src/errors.rs | 2 + beacon_node/execution_layer/src/lib.rs | 26 ++- .../http_api/src/build_block_contents.rs | 37 +++- beacon_node/http_api/src/lib.rs | 8 +- common/eth2/src/types.rs | 46 ++++- consensus/types/src/blob_sidecar.rs | 1 + 9 files changed, 223 insertions(+), 97 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index d410e860db0..692e7174a75 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -66,7 +66,10 @@ use crate::validator_monitor::{ }; use crate::validator_pubkey_cache::ValidatorPubkeyCache; use crate::{metrics, BeaconChainError, BeaconForkChoiceStore, BeaconSnapshot, CachedHead}; -use eth2::types::{EventKind, SseBlock, SseExtendedPayloadAttributes, SyncDuty}; +use eth2::types::{ + BlobRootsWrapper, BlobsOrBlobRoots, BlobsWrapper, EventKind, SseBlock, + SseExtendedPayloadAttributes, SyncDuty, +}; use execution_layer::{ BlockProposalContents, BuilderParams, ChainHealth, ExecutionLayer, FailedCondition, PayloadAttributes, PayloadStatus, @@ -115,9 +118,8 @@ use store::{ use task_executor::{ShutdownReason, TaskExecutor}; use tokio_stream::Stream; use tree_hash::TreeHash; -use types::beacon_block_body::KzgCommitments; use types::beacon_state::CloneConfig; -use types::blob_sidecar::{BlobSidecarList, Blobs}; +use types::blob_sidecar::{BlindedBlobSidecar, BlindedBlobSidecarList, BlobSidecarList}; use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; use types::*; @@ -465,7 +467,8 @@ pub struct BeaconChain { pub validator_monitor: RwLock>, /// The slot at which blocks are downloaded back to. pub genesis_backfill_slot: Slot, - pub proposal_blob_cache: BlobCache, + pub proposal_blob_cache: BlobCache>, + pub proposal_blinded_blob_cache: BlobCache>, pub data_availability_checker: Arc>, pub kzg: Option>, } @@ -4735,7 +4738,7 @@ impl BeaconChain { bls_to_execution_changes, } = partial_beacon_block; - let (inner_block, blobs_opt, proofs_opt) = match &state { + let (inner_block, blobs_or_blobs_root_opt, proofs_opt) = match &state { BeaconState::Base(_) => ( BeaconBlock::Base(BeaconBlockBase { slot, @@ -4842,7 +4845,7 @@ impl BeaconChain { ) } BeaconState::Deneb(_) => { - let (payload, kzg_commitments, blobs, proofs) = block_contents + let (payload, kzg_commitments, blobs_or_blob_roots, proofs) = block_contents .ok_or(BlockProductionError::MissingExecutionPayload)? .deconstruct(); ( @@ -4870,7 +4873,7 @@ impl BeaconChain { .ok_or(BlockProductionError::InvalidPayloadFork)?, }, }), - blobs, + blobs_or_blob_roots, proofs, ) } @@ -4924,7 +4927,7 @@ impl BeaconChain { //FIXME(sean) // - add a new timer for processing here - if let Some(blobs) = blobs_opt { + if let Some(blobs_or_blob_roots) = blobs_or_blobs_root_opt { let kzg = self .kzg .as_ref() @@ -4936,58 +4939,89 @@ impl BeaconChain { ) })?; - if expected_kzg_commitments.len() != blobs.len() { + if expected_kzg_commitments.len() != blobs_or_blob_roots.len() { return Err(BlockProductionError::MissingKzgCommitment(format!( "Missing KZG commitment for slot {}. Expected {}, got: {}", slot, - blobs.len(), + blobs_or_blob_roots.len(), expected_kzg_commitments.len() ))); } - let kzg_proofs = if let Some(proofs) = proofs_opt { - Vec::from(proofs) - } else { - Self::compute_blob_kzg_proofs(kzg, &blobs, expected_kzg_commitments, slot)? - }; + let kzg_proofs = proofs_opt.ok_or(BlockProductionError::MissingKzgProof)?; - kzg_utils::validate_blobs::( - kzg, - expected_kzg_commitments, - &blobs, - &kzg_proofs, - ) - .map_err(BlockProductionError::KzgError)?; - - let blob_sidecars = BlobSidecarList::from( - blobs - .into_iter() - .enumerate() - .map(|(blob_index, blob)| { - let kzg_commitment = expected_kzg_commitments - .get(blob_index) - .expect("KZG commitment should exist for blob"); - - let kzg_proof = kzg_proofs - .get(blob_index) - .expect("KZG proof should exist for blob"); - - Ok(Arc::new(BlobSidecar { - block_root: beacon_block_root, - index: blob_index as u64, - slot, - block_parent_root: block.parent_root(), - proposer_index, - blob, - kzg_commitment: *kzg_commitment, - kzg_proof: *kzg_proof, - })) - }) - .collect::, BlockProductionError>>()?, - ); + match blobs_or_blob_roots { + BlobsOrBlobRoots::Blobs(BlobsWrapper { blobs }) => { + kzg_utils::validate_blobs::( + kzg, + expected_kzg_commitments, + &blobs, + &kzg_proofs, + ) + .map_err(BlockProductionError::KzgError)?; + + let blob_sidecars = BlobSidecarList::from( + blobs + .into_iter() + .enumerate() + .map(|(blob_index, blob)| { + let kzg_commitment = expected_kzg_commitments + .get(blob_index) + .expect("KZG commitment should exist for blob"); + + let kzg_proof = kzg_proofs + .get(blob_index) + .expect("KZG proof should exist for blob"); + + Ok(Arc::new(BlobSidecar { + block_root: beacon_block_root, + index: blob_index as u64, + slot, + block_parent_root: block.parent_root(), + proposer_index, + blob, + kzg_commitment: *kzg_commitment, + kzg_proof: *kzg_proof, + })) + }) + .collect::, BlockProductionError>>()?, + ); - self.proposal_blob_cache - .put(beacon_block_root, blob_sidecars); + self.proposal_blob_cache + .put(beacon_block_root, blob_sidecars); + } + BlobsOrBlobRoots::BlobRoots(BlobRootsWrapper { blob_roots }) => { + let blinded_blob_sidecars = BlindedBlobSidecarList::::from( + blob_roots + .into_iter() + .enumerate() + .map(|(blob_index, blob_root)| { + let kzg_commitment = expected_kzg_commitments + .get(blob_index) + .expect("KZG commitment should exist for blob"); + + let kzg_proof = kzg_proofs + .get(blob_index) + .expect("KZG proof should exist for blob"); + + Ok(Arc::new(BlindedBlobSidecar { + block_root: beacon_block_root, + index: blob_index as u64, + slot, + block_parent_root: block.parent_root(), + proposer_index, + blob_root, + kzg_commitment: *kzg_commitment, + kzg_proof: *kzg_proof, + })) + }) + .collect::, BlockProductionError>>()?, + ); + + self.proposal_blinded_blob_cache + .put(beacon_block_root, blinded_blob_sidecars); + } + } } metrics::inc_counter(&metrics::BLOCK_PRODUCTION_SUCCESSES); @@ -5003,28 +5037,29 @@ impl BeaconChain { Ok((block, state)) } - fn compute_blob_kzg_proofs( - kzg: &Arc, - blobs: &Blobs, - expected_kzg_commitments: &KzgCommitments, - slot: Slot, - ) -> Result, BlockProductionError> { - blobs - .iter() - .enumerate() - .map(|(blob_index, blob)| { - let kzg_commitment = expected_kzg_commitments.get(blob_index).ok_or( - BlockProductionError::MissingKzgCommitment(format!( - "Missing KZG commitment for slot {} blob index {}", - slot, blob_index - )), - )?; - - kzg_utils::compute_blob_kzg_proof::(kzg, blob, *kzg_commitment) - .map_err(BlockProductionError::KzgError) - }) - .collect::, BlockProductionError>>() - } + // FIXME(jimmy) when do we need this? + // fn compute_blob_kzg_proofs( + // kzg: &Arc, + // blobs: &Blobs, + // expected_kzg_commitments: &KzgCommitments, + // slot: Slot, + // ) -> Result, BlockProductionError> { + // blobs + // .iter() + // .enumerate() + // .map(|(blob_index, blob)| { + // let kzg_commitment = expected_kzg_commitments.get(blob_index).ok_or( + // BlockProductionError::MissingKzgCommitment(format!( + // "Missing KZG commitment for slot {} blob index {}", + // slot, blob_index + // )), + // )?; + // + // kzg_utils::compute_blob_kzg_proof::(kzg, blob, *kzg_commitment) + // .map_err(BlockProductionError::KzgError) + // }) + // .collect::, BlockProductionError>>() + // } /// This method must be called whenever an execution engine indicates that a payload is /// invalid. diff --git a/beacon_node/beacon_chain/src/blob_cache.rs b/beacon_node/beacon_chain/src/blob_cache.rs index 64f113c285c..58411d226d8 100644 --- a/beacon_node/beacon_chain/src/blob_cache.rs +++ b/beacon_node/beacon_chain/src/blob_cache.rs @@ -1,18 +1,18 @@ use lru::LruCache; use parking_lot::Mutex; -use types::{BlobSidecarList, EthSpec, Hash256}; +use types::Hash256; pub const DEFAULT_BLOB_CACHE_SIZE: usize = 10; /// A cache blobs by beacon block root. -pub struct BlobCache { - blobs: Mutex>>, +pub struct BlobCache { + blobs: Mutex>, } #[derive(Hash, PartialEq, Eq)] struct BlobCacheId(Hash256); -impl Default for BlobCache { +impl Default for BlobCache { fn default() -> Self { BlobCache { blobs: Mutex::new(LruCache::new(DEFAULT_BLOB_CACHE_SIZE)), @@ -20,16 +20,12 @@ impl Default for BlobCache { } } -impl BlobCache { - pub fn put( - &self, - beacon_block: Hash256, - blobs: BlobSidecarList, - ) -> Option> { +impl BlobCache { + pub fn put(&self, beacon_block: Hash256, blobs: T) -> Option { self.blobs.lock().put(BlobCacheId(beacon_block), blobs) } - pub fn pop(&self, root: &Hash256) -> Option> { + pub fn pop(&self, root: &Hash256) -> Option { self.blobs.lock().pop(&BlobCacheId(*root)) } } diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index a35e7d615d8..4de0c0b4bfe 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -891,6 +891,7 @@ where .map_err(|e| format!("Error initializing DataAvailabiltyChecker: {:?}", e))?, ), proposal_blob_cache: BlobCache::default(), + proposal_blinded_blob_cache: BlobCache::default(), kzg, }; diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index e4c4ff2517c..74eb4841276 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -273,6 +273,7 @@ pub enum BlockProductionError { payload_block_hash: ExecutionBlockHash, }, NoBlobsCached, + NoBlindedBlobsCached, FailedToReadFinalizedBlock(store::Error), MissingFinalizedBlock(Hash256), BlockTooLarge(usize), @@ -280,6 +281,7 @@ pub enum BlockProductionError { MissingSyncAggregate, MissingExecutionPayload, MissingKzgCommitment(String), + MissingKzgProof, TokioJoin(tokio::task::JoinError), BeaconChain(BeaconChainError), InvalidPayloadFork, diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index 999e6036e41..ad1e78d4611 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -13,7 +13,7 @@ pub use engine_api::*; pub use engine_api::{http, http::deposit_methods, http::HttpJsonRpc}; use engines::{Engine, EngineError}; pub use engines::{EngineState, ForkchoiceState}; -use eth2::types::{builder_bid::SignedBuilderBid, ForkVersionedResponse}; +use eth2::types::{builder_bid::SignedBuilderBid, BlobsOrBlobRoots, ForkVersionedResponse}; use ethers_core::abi::ethereum_types::FromStrRadixErr; use ethers_core::types::transaction::eip2930::AccessListItem; use ethers_core::types::{Transaction as EthersTransaction, U64}; @@ -206,7 +206,7 @@ impl> BlockProposalContents ( Payload, Option>, - Option>, + Option>, Option>, ) { match self { @@ -220,14 +220,24 @@ impl> BlockProposalContents (payload, Some(kzg_commitments), Some(blobs), Some(proofs)), + } => ( + payload, + Some(kzg_commitments), + Some(BlobsOrBlobRoots::blobs(blobs)), + Some(proofs), + ), Self::PayloadAndBlindedBlobs { - payload: _, + payload, block_value: _, - kzg_commitments: _, - blob_roots: _, - proofs: _, - } => unimplemented!("need to work out how to handle blinded blobs"), + kzg_commitments, + blob_roots, + proofs, + } => ( + payload, + Some(kzg_commitments), + Some(BlobsOrBlobRoots::blob_roots(blob_roots)), + Some(proofs), + ), } } diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index d6c5ada0071..f9f932707f7 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -1,5 +1,8 @@ use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProductionError}; -use eth2::types::{BeaconBlockAndBlobSidecars, BlockContents}; +use eth2::types::{ + BeaconBlockAndBlobSidecars, BlindedBeaconBlockAndBlobSidecars, BlindedBlockContents, + BlockContents, +}; use std::sync::Arc; use types::{AbstractExecPayload, BeaconBlock, ForkName}; @@ -31,3 +34,35 @@ pub fn build_block_contents, +>( + fork_name: ForkName, + chain: Arc>, + blinded_block: BeaconBlock, +) -> Result, Error> { + match fork_name { + ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + Ok(BlindedBlockContents::Block(blinded_block)) + } + ForkName::Deneb => { + let block_root = &blinded_block.canonical_root(); + if let Some(blinded_blob_sidecars) = chain.proposal_blinded_blob_cache.pop(block_root) { + let blinded_block_and_blobs = BlindedBeaconBlockAndBlobSidecars { + blinded_block, + blinded_blob_sidecars, + }; + + Ok(BlindedBlockContents::BlockAndBlobSidecars( + blinded_block_and_blobs, + )) + } else { + Err(warp_utils::reject::block_production_error( + BlockProductionError::NoBlindedBlobsCached, + )) + } + } + } +} diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 3edc6aa3012..8521f7bc390 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -2634,9 +2634,9 @@ pub fn serve( .map_err(inconsistent_fork_rejection)?; let block_contents = - build_block_contents::build_block_contents(fork_name, chain, block); + build_block_contents::build_block_contents(fork_name, chain, block)?; - fork_versioned_response(endpoint_version, fork_name, block_contents?) + fork_versioned_response(endpoint_version, fork_name, block_contents) .map(|response| warp::reply::json(&response).into_response()) }, ); @@ -2692,8 +2692,10 @@ pub fn serve( .fork_name(&chain.spec) .map_err(inconsistent_fork_rejection)?; + let block_contents = build_block_contents::build_blinded_block_contents(fork_name, chain, block)?; + // Pose as a V2 endpoint so we return the fork `version`. - fork_versioned_response(V2, fork_name, block) + fork_versioned_response(V2, fork_name, block_contents) .map(|response| warp::reply::json(&response).into_response()) }, ); diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index 834886ce9cc..6445257fc95 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -11,7 +11,7 @@ use std::convert::TryFrom; use std::fmt; use std::str::{from_utf8, FromStr}; use std::time::Duration; -use types::blob_sidecar::BlindedBlobSidecarList; +use types::blob_sidecar::{BlindedBlobSidecarList, BlobRoots, Blobs}; pub use types::*; #[cfg(feature = "lighthouse")] @@ -1274,6 +1274,50 @@ mod tests { } } +#[derive(Debug, Serialize, Deserialize)] +#[serde(bound = "T: EthSpec")] +pub struct BlobsWrapper { + pub blobs: Blobs, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(bound = "T: EthSpec")] +pub struct BlobRootsWrapper { + pub blob_roots: BlobRoots, +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(untagged)] +#[serde(bound = "T: EthSpec")] +pub enum BlobsOrBlobRoots { + Blobs(BlobsWrapper), + BlobRoots(BlobRootsWrapper), +} + +impl BlobsOrBlobRoots { + pub fn blob_roots(blob_roots: BlobRoots) -> Self { + Self::BlobRoots(BlobRootsWrapper { blob_roots }) + } + + pub fn blobs(blobs: Blobs) -> Self { + Self::Blobs(BlobsWrapper { blobs }) + } + + pub fn len(&self) -> usize { + match self { + Self::Blobs(BlobsWrapper { blobs }) => blobs.len(), + Self::BlobRoots(BlobRootsWrapper { blob_roots }) => blob_roots.len(), + } + } + + pub fn is_empty(&self) -> bool { + match self { + Self::Blobs(BlobsWrapper { blobs }) => blobs.is_empty(), + Self::BlobRoots(BlobRootsWrapper { blob_roots }) => blob_roots.is_empty(), + } + } +} + /// A wrapper over a [`BeaconBlock`] or a [`BeaconBlockAndBlobSidecars`]. #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(untagged)] diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 712b69a0b85..39ff34d84ac 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -80,6 +80,7 @@ pub type BlobSidecarList = VariableList>, :: pub type FixedBlobSidecarList = FixedVector>>, ::MaxBlobsPerBlock>; pub type Blobs = VariableList, ::MaxBlobsPerBlock>; +pub type BlobRoots = VariableList::MaxBlobsPerBlock>; impl SignedRoot for BlobSidecar {}