Merge branch 'eip4844' into builder-bid-4844

# Conflicts:
#	beacon_node/http_api/src/lib.rs

.github/workflows/docker.yml

@@ -6,6 +6,7 @@ on:
             - unstable
             - stable
             - capella
+            - eip4844
         tags:
             - v*
 
@@ -40,6 +41,12 @@ jobs:
               run: |
                     echo "VERSION=capella" >> $GITHUB_ENV
                     echo "VERSION_SUFFIX=" >> $GITHUB_ENV
+                    echo "CROSS_FEATURES=withdrawals-processing" >> $GITHUB_ENV
+            - name: Extract version (if eip4844)
+              if: github.event.ref == 'refs/heads/eip4844'
+              run: |
+                    echo "VERSION=eip4844" >> $GITHUB_ENV
+                    echo "VERSION_SUFFIX=" >> $GITHUB_ENV
             - name: Extract version (if tagged release)
               if: startsWith(github.event.ref, 'refs/tags')
               run: |
@@ -48,6 +55,7 @@ jobs:
         outputs:
             VERSION: ${{ env.VERSION }}
             VERSION_SUFFIX: ${{ env.VERSION_SUFFIX }}
+            CROSS_FEATURES: ${{ env.CROSS_FEATURES }}
     build-docker-single-arch:
         name: build-docker-${{ matrix.binary }}
         runs-on: ubuntu-22.04
@@ -66,7 +74,7 @@ jobs:
             DOCKER_CLI_EXPERIMENTAL: enabled
             VERSION: ${{ needs.extract-version.outputs.VERSION }}
             VERSION_SUFFIX: ${{ needs.extract-version.outputs.VERSION_SUFFIX }}
-            CROSS_FEATURES: withdrawals,withdrawals-processing
+            CROSS_FEATURES: ${{ needs.extract-version.outputs.CROSS_FEATURES }}
         steps:
             - uses: actions/checkout@v3
             - name: Update Rust

Dockerfile

@@ -1,5 +1,5 @@
 FROM rust:1.65.0-bullseye AS builder
-RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake libclang-dev protobuf-compiler
+RUN apt-get update && apt-get -y upgrade && apt-get install -y cmake clang libclang-dev protobuf-compiler
 COPY . lighthouse
 ARG FEATURES
 ENV FEATURES $FEATURES

Makefile

@@ -21,7 +21,7 @@ CROSS_FEATURES ?= gnosis,slasher-lmdb,slasher-mdbx
 CROSS_PROFILE ?= release
 
 # List of features to use when running EF tests.
-EF_TEST_FEATURES ?= withdrawals,withdrawals-processing
+EF_TEST_FEATURES ?= beacon_chain/withdrawals-processing
 
 # Cargo profile for regular builds.
 PROFILE ?= release

beacon_node/Cargo.toml

@@ -13,7 +13,6 @@ node_test_rig = { path = "../testing/node_test_rig" }
 
 [features]
 write_ssz_files = ["beacon_chain/write_ssz_files"]  # Writes debugging .ssz files to /tmp during block processing.
-withdrawals = ["beacon_chain/withdrawals", "types/withdrawals", "store/withdrawals", "execution_layer/withdrawals"]
 withdrawals-processing = [
     "beacon_chain/withdrawals-processing",
     "store/withdrawals-processing",

beacon_node/beacon_chain/Cargo.toml

@@ -10,7 +10,6 @@ default = ["participation_metrics"]
 write_ssz_files = []  # Writes debugging .ssz files to /tmp during block processing.
 participation_metrics = []  # Exposes validator participation metrics to Prometheus.
 fork_from_env = [] # Initialise the harness chain spec from the FORK_NAME env variable
-withdrawals = ["state_processing/withdrawals", "types/withdrawals", "store/withdrawals", "execution_layer/withdrawals"]
 withdrawals-processing = [
     "state_processing/withdrawals-processing",
     "store/withdrawals-processing",

beacon_node/beacon_chain/src/attester_cache.rs

@@ -42,6 +42,7 @@ pub enum Error {
     // Boxed to avoid an infinite-size recursion issue.
     BeaconChain(Box<BeaconChainError>),
     MissingBeaconState(Hash256),
+    MissingBlobs,
     FailedToTransitionState(StateAdvanceError),
     CannotAttestToFutureState {
         state_slot: Slot,

beacon_node/beacon_chain/src/beacon_chain.rs

@@ -62,7 +62,7 @@ use crate::{metrics, BeaconChainError, BeaconForkChoiceStore, BeaconSnapshot, Ca
 use eth2::types::{EventKind, SseBlock, SyncDuty};
 use execution_layer::{
     BlockProposalContents, BuilderParams, ChainHealth, ExecutionLayer, FailedCondition,
-    PayloadAttributes, PayloadAttributesV2, PayloadStatus,
+    PayloadAttributes, PayloadStatus,
 };
 pub use fork_choice::CountUnrealized;
 use fork_choice::{
@@ -80,14 +80,12 @@ use slasher::Slasher;
 use slog::{crit, debug, error, info, trace, warn, Logger};
 use slot_clock::SlotClock;
 use ssz::Encode;
-#[cfg(feature = "withdrawals")]
-use state_processing::per_block_processing::get_expected_withdrawals;
 use state_processing::{
     common::get_attesting_indices_from_state,
     per_block_processing,
     per_block_processing::{
-        errors::AttestationValidationError, verify_attestation_for_block_inclusion,
-        VerifySignatures,
+        errors::AttestationValidationError, get_expected_withdrawals,
+        verify_attestation_for_block_inclusion, VerifySignatures,
     },
     per_slot_processing,
     state_advance::{complete_state_advance, partial_state_advance},
@@ -290,7 +288,6 @@ struct PartialBeaconBlock<E: EthSpec, Payload: AbstractExecPayload<E>> {
     voluntary_exits: Vec<SignedVoluntaryExit>,
     sync_aggregate: Option<SyncAggregate<E>>,
     prepare_payload_handle: Option<PreparePayloadHandle<E, Payload>>,
-    #[cfg(feature = "withdrawals")]
     bls_to_execution_changes: Vec<SignedBlsToExecutionChange>,
 }
 
@@ -2938,7 +2935,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         // If the write fails, revert fork choice to the version from disk, else we can
         // end up with blocks in fork choice that are missing from disk.
         // See https://github.com/sigp/lighthouse/issues/2028
-        let (signed_block, blobs) = signed_block.deconstruct();
+        let (signed_block, blobs) = signed_block.deconstruct(Some(block_root));
         let block = signed_block.message();
         let mut ops: Vec<_> = confirmed_state_roots
             .into_iter()
@@ -2947,7 +2944,9 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         ops.push(StoreOp::PutBlock(block_root, signed_block.clone()));
         ops.push(StoreOp::PutState(block.state_root(), &state));
 
-        if let Some(blobs) = blobs {
+        if let Some(blobs) = blobs? {
+            //FIXME(sean) using this for debugging for now
+            info!(self.log, "Writing blobs to store"; "block_root" => ?block_root);
             ops.push(StoreOp::PutBlobs(block_root, blobs));
         };
         let txn_lock = self.store.hot_db.begin_rw_transaction();
@@ -4206,7 +4205,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
         let eth1_data = eth1_chain.eth1_data_for_block_production(&state, &self.spec)?;
         let deposits = eth1_chain.deposits_for_block_inclusion(&state, &eth1_data, &self.spec)?;
 
-        #[cfg(feature = "withdrawals")]
         let bls_to_execution_changes = self
             .op_pool
             .get_bls_to_execution_changes(&state, &self.spec);
@@ -4369,7 +4367,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             voluntary_exits,
             sync_aggregate,
             prepare_payload_handle,
-            #[cfg(feature = "withdrawals")]
             bls_to_execution_changes,
         })
     }
@@ -4398,7 +4395,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             // this function. We can assume that the handle has already been consumed in order to
             // produce said `execution_payload`.
             prepare_payload_handle: _,
-            #[cfg(feature = "withdrawals")]
             bls_to_execution_changes,
         } = partial_beacon_block;
 
@@ -4499,7 +4495,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                             execution_payload: payload
                                 .try_into()
                                 .map_err(|_| BlockProductionError::InvalidPayloadFork)?,
-                            #[cfg(feature = "withdrawals")]
                             bls_to_execution_changes: bls_to_execution_changes.into(),
                         },
                     }),
@@ -4531,7 +4526,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
                             execution_payload: payload
                                 .try_into()
                                 .map_err(|_| BlockProductionError::InvalidPayloadFork)?,
-                            #[cfg(feature = "withdrawals")]
                             bls_to_execution_changes: bls_to_execution_changes.into(),
                             blob_kzg_commitments: kzg_commitments
                                 .ok_or(BlockProductionError::InvalidPayloadFork)?,
@@ -4831,7 +4825,6 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             return Ok(());
         }
 
-        #[cfg(feature = "withdrawals")]
         let withdrawals = match self.spec.fork_name_at_slot::<T::EthSpec>(prepare_slot) {
             ForkName::Base | ForkName::Altair | ForkName::Merge => None,
             ForkName::Capella | ForkName::Eip4844 => {
@@ -4866,10 +4859,7 @@ impl<T: BeaconChainTypes> BeaconChain<T> {
             execution_layer
                 .get_suggested_fee_recipient(proposer as u64)
                 .await,
-            #[cfg(feature = "withdrawals")]
             withdrawals,
-            #[cfg(not(feature = "withdrawals"))]
-            None,
         );
 
         debug!(

beacon_node/beacon_chain/src/blob_cache.rs

@@ -1,7 +1,6 @@
 use lru::LruCache;
 use parking_lot::Mutex;
-use tree_hash::TreeHash;
-use types::{BlobsSidecar, EthSpec, ExecutionPayload, Hash256};
+use types::{BlobsSidecar, EthSpec, Hash256};
 
 pub const DEFAULT_BLOB_CACHE_SIZE: usize = 10;
 

beacon_node/beacon_chain/src/blob_verification.rs

@@ -2,9 +2,8 @@ use slot_clock::SlotClock;
 
 use crate::beacon_chain::{BeaconChain, BeaconChainTypes, MAXIMUM_GOSSIP_CLOCK_DISPARITY};
 use crate::{kzg_utils, BeaconChainError};
-use bls::PublicKey;
 use state_processing::per_block_processing::eip4844::eip4844::verify_kzg_commitments_against_transactions;
-use types::consts::eip4844::BLS_MODULUS;
+use types::signed_beacon_block::BlobReconstructionError;
 use types::{BeaconStateError, BlobsSidecar, Hash256, KzgCommitment, Slot, Transactions};
 
 #[derive(Debug)]
@@ -87,6 +86,14 @@ pub enum BlobError {
     /// We were unable to process this sync committee message due to an internal error. It's unclear if the
     /// sync committee message is valid.
     BeaconChainError(BeaconChainError),
+    /// No blobs for the specified block where we would expect blobs.
+    MissingBlobs,
+}
+
+impl From<BlobReconstructionError> for BlobError {
+    fn from(_: BlobReconstructionError) -> Self {
+        BlobError::MissingBlobs
+    }
 }
 
 impl From<BeaconChainError> for BlobError {