Add blob sidecar inclusion proof

pysetup/spec_builders/deneb.py

@@ -23,7 +23,15 @@ def sundry_functions(cls) -> str:
         return '''
 def retrieve_blobs_and_proofs(beacon_block_root: Root) -> Tuple[Sequence[Blob], Sequence[KZGProof]]:
     # pylint: disable=unused-argument
-    return [], []'''
+    return [], []
+
+
+def compute_commitment_inclusion_proof(
+    body: BeaconBlockBody,
+    index: GeneralizedIndex
+) -> Sequence[Bytes32]:
+    return build_proof(body.get_backing(), index)
+'''
 
     @classmethod
     def execution_engine_cls(cls) -> str:
@@ -68,4 +76,5 @@ def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
             'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value,
             'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value,
             'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value,
+            'MAX_BLOB_COMMITMENTS_PER_BLOCK': spec_object.preset_vars['MAX_BLOB_COMMITMENTS_PER_BLOCK'].value,
         }

specs/deneb/beacon-chain.md

@@ -187,6 +187,19 @@ def kzg_commitment_to_versioned_hash(kzg_commitment: KZGCommitment) -> Versioned
     return VERSIONED_HASH_VERSION_KZG + hash(kzg_commitment)[1:]
 ```
 
+#### `is_valid_merkle_path`
+
+```python
+def is_valid_merkle_path(leaf: Bytes32, branch: Sequence[Bytes32], gindex: int, root: Root) -> bool:
+    value = leaf
+    for i in range(len(branch)):
+        if (gindex >> i) & 1 == 0:
+            value = hash(branch[i] + value)
+        else:
+            value = hash(value + branch[i])
+    return value == root
+```
+
 ### Beacon state accessors
 
 #### Modified `get_attestation_participation_flag_indices`

specs/deneb/p2p-interface.md

@@ -14,10 +14,10 @@ The specification of these changes continues in the same format as the network s
   - [Configuration](#configuration)
   - [Containers](#containers)
     - [`BlobSidecar`](#blobsidecar)
-    - [`SignedBlobSidecar`](#signedblobsidecar)
     - [`BlobIdentifier`](#blobidentifier)
     - [Helpers](#helpers)
-      - [`verify_blob_sidecar_signature`](#verify_blob_sidecar_signature)
+      - [`verify_blob_sidecar_inclusion_proof`](#verify_blob_sidecar_inclusion_proof)
+    - [`is_valid_merkle_path`](#is_valid_merkle_path)
   - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub)
     - [Topics and messages](#topics-and-messages)
       - [Global topics](#global-topics)
@@ -51,6 +51,8 @@ The specification of these changes continues in the same format as the network s
 | `MAX_REQUEST_BLOB_SIDECARS`              | `MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK` | Maximum number of blob sidecars in a single request  |
 | `MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS`  | `2**12` (= 4096 epochs, ~18 days) | The minimum epoch range over which a node must serve blob sidecars  |
 | `BLOB_SIDECAR_SUBNET_COUNT`              | `6`                               | The number of blob sidecar subnets used in the gossipsub protocol.  |
+| `BLOB_KZG_COMMITMENTS_GINDEX`            | `get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')`  (= 27) | `blob_kzg_commitments` field gindex on `BeaconBlockBody` container  |
+| `KZG_COMMITMENT_INCLUSION_PROOF_DEPTH`   | `floorlog2(BLOB_KZG_COMMITMENTS_GINDEX) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)  # noqa: E501` | Merkle proof for `blob_kzg_commitments` list item |
 
 ### Containers
 
@@ -60,24 +62,12 @@ The specification of these changes continues in the same format as the network s
 
 ```python
 class BlobSidecar(Container):
-    block_root: Root
     index: BlobIndex  # Index of blob in block
-    slot: Slot
-    block_parent_root: Root  # Proposer shuffling determinant
-    proposer_index: ValidatorIndex
     blob: Blob
     kzg_commitment: KZGCommitment
     kzg_proof: KZGProof  # Allows for quick verification of kzg_commitment
-```
-
-#### `SignedBlobSidecar`
-
-*[New in Deneb:EIP4844]*
-
-```python
-class SignedBlobSidecar(Container):
-    message: BlobSidecar
-    signature: BLSSignature
+    commitment_inclusion_proof: List[Bytes32, KZG_COMMITMENT_INCLUSION_PROOF_DEPTH]
+    signed_block_header: SignedBeaconBlockHeader
 ```
 
 #### `BlobIdentifier`
@@ -92,13 +82,16 @@ class BlobIdentifier(Container):
 
 #### Helpers
 
-##### `verify_blob_sidecar_signature`
+##### `verify_blob_sidecar_inclusion_proof`
 
 ```python
-def verify_blob_sidecar_signature(state: BeaconState, signed_blob_sidecar: SignedBlobSidecar) -> bool:
-    proposer = state.validators[signed_blob_sidecar.message.proposer_index]
-    signing_root = compute_signing_root(signed_blob_sidecar.message, get_domain(state, DOMAIN_BLOB_SIDECAR))
-    return bls.Verify(proposer.pubkey, signing_root, signed_blob_sidecar.signature)
+def verify_blob_sidecar_inclusion_proof(blob_sidecar: BlobSidecar) -> bool:
+    return is_valid_merkle_path(
+        leaf=blob_sidecar.kzg_commitment.hash_tree_root(),
+        branch=blob_sidecar.commitment_inclusion_proof,
+        gindex=get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments', blob_sidecar.index),
+        root=blob_sidecar.signed_block_header.message.body_root,
+    )
 ```
 
 ### The gossip domain: gossipsub
@@ -123,7 +116,7 @@ The new topics along with the type of the `data` field of a gossipsub message ar
 
 | Name | Message Type |
 | - | - |
-| `blob_sidecar_{subnet_id}` | `SignedBlobSidecar` [New in Deneb:EIP4844] |
+| `blob_sidecar_{subnet_id}` | `BlobSidecar` [New in Deneb:EIP4844] |
 
 ##### Global topics
 
@@ -146,18 +139,19 @@ New validation:
 
 This topic is used to propagate signed blob sidecars, where each blob index maps to some `subnet_id`.
 
-The following validations MUST pass before forwarding the `signed_blob_sidecar` on the network, assuming the alias `sidecar = signed_blob_sidecar.message`:
+The following validations MUST pass before forwarding the `blob_sidecar` on the network, assuming the alias `sidecar = blob_sidecar` and `block_header = blob_sidecar.signed_block_header.message`:
 
 - _[REJECT]_ The sidecar's index is consistent with `MAX_BLOBS_PER_BLOCK` -- i.e. `sidecar.index < MAX_BLOBS_PER_BLOCK`.
 - _[REJECT]_ The sidecar is for the correct subnet -- i.e. `compute_subnet_for_blob_sidecar(sidecar.index) == subnet_id`.
-- _[IGNORE]_ The sidecar is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `sidecar.slot <= current_slot` (a client MAY queue future sidecars for processing at the appropriate slot).
-- _[IGNORE]_ The sidecar is from a slot greater than the latest finalized slot -- i.e. validate that `sidecar.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)`
-- _[IGNORE]_ The sidecar's block's parent (defined by `sidecar.block_parent_root`) has been seen (via both gossip and non-gossip sources) (a client MAY queue sidecars for processing once the parent block is retrieved).
-- _[REJECT]_ The sidecar's block's parent (defined by `sidecar.block_parent_root`) passes validation.
-- _[REJECT]_ The sidecar is from a higher slot than the sidecar's block's parent (defined by `sidecar.block_parent_root`).
-- _[REJECT]_ The proposer signature, `signed_blob_sidecar.signature`, is valid as verified by `verify_blob_sidecar_signature`.
-- _[IGNORE]_ The sidecar is the only sidecar with valid signature received for the tuple `(sidecar.block_root, sidecar.index)`.
-- _[REJECT]_ The sidecar is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `block_parent_root`/`slot`).
+- _[IGNORE]_ The sidecar is not from a future slot (with a `MAXIMUM_GOSSIP_CLOCK_DISPARITY` allowance) -- i.e. validate that `block_header.slot <= current_slot` (a client MAY queue future sidecars for processing at the appropriate slot).
+- _[IGNORE]_ The sidecar is from a slot greater than the latest finalized slot -- i.e. validate that `block_header.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch)`
+- _[IGNORE]_ The sidecar's block's parent (defined by `block_header.parent_root`) has been seen (via both gossip and non-gossip sources) (a client MAY queue sidecars for processing once the parent block is retrieved).
+- _[REJECT]_ The sidecar's block's parent (defined by `block_header.parent_root`) passes validation.
+- _[REJECT]_ The sidecar is from a higher slot than the sidecar's block's parent (defined by `block_header.parent_root`).
+- _[REJECT]_ The proposer signature in `blob_sidecar.signed_block_header`, is valid with respect to the `proposer_index` pubkey.
+- _[REJECT]_ The sidecar's inclusion proof is valid as verified by `verify_blob_sidecar_inclusion_proof`.
+- _[IGNORE]_ The sidecar is the only sidecar with valid signature received for the tuple `(hash_tree_root(block_header), sidecar.index)`.
+- _[REJECT]_ The sidecar is proposed by the expected `proposer_index` for the block's slot in the context of the current shuffling (defined by `block_header.parent_root`/`block_header.slot`).
   If the `proposer_index` cannot immediately be verified against the expected shuffling, the sidecar MAY be queued for later processing while proposers for the block's branch are calculated -- in such a case _do not_ `REJECT`, instead `IGNORE` this message.
 
 ###### `beacon_aggregate_and_proof`

specs/deneb/validator.md

@@ -21,7 +21,7 @@
     - [Constructing the `BeaconBlockBody`](#constructing-the-beaconblockbody)
       - [ExecutionPayload](#executionpayload)
       - [Blob KZG commitments](#blob-kzg-commitments)
-    - [Constructing the `SignedBlobSidecar`s](#constructing-the-signedblobsidecars)
+    - [Constructing the `BlobSidecar`s](#constructing-the-blobsidecars)
       - [Sidecar](#sidecar)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
@@ -133,47 +133,44 @@ use the `payload_id` to retrieve `blobs`, `blob_kzg_commitments`, and `blob_kzg_
 via `get_payload(payload_id).blobs_bundle`.
 2. Set `block.body.blob_kzg_commitments = blob_kzg_commitments`.
 
-#### Constructing the `SignedBlobSidecar`s
+#### Constructing the `BlobSidecar`s
 
 *[New in Deneb:EIP4844]*
 
-To construct a `SignedBlobSidecar`, a `signed_blob_sidecar` is defined with the necessary context for block and sidecar proposal.
+To construct a `BlobSidecar`, a `blob_sidecar` is defined with the necessary context for block and sidecar proposal.
 
 ##### Sidecar
 
-Blobs associated with a block are packaged into sidecar objects for distribution to the network.
+Blobs associated with a block are packaged into sidecar objects for distribution to the associated sidecar topic, the `blob_sidecar_{subnet_id}` pubsub topic.
 
 Each `sidecar` is obtained from:
 ```python
-def get_blob_sidecars(block: BeaconBlock,
+def get_blob_sidecars(signed_block: SignedBeaconBlock,
                       blobs: Sequence[Blob],
                       blob_kzg_proofs: Sequence[KZGProof]) -> Sequence[BlobSidecar]:
+    block = signed_block.message
+    block_header = BeaconBlockHeader(
+        slot=block.slot,
+        proposer_index=block.proposer_index,
+        parent_root=block.parent_root,
+        state_root=block.state_root,
+        body_root=hash_tree_root(block.body),
+    )
+    signed_block_header = SignedBeaconBlockHeader(message=block_header, signature=signed_block.signature)
     return [
         BlobSidecar(
-            block_root=hash_tree_root(block),
             index=index,
-            slot=block.slot,
-            block_parent_root=block.parent_root,
             blob=blob,
             kzg_commitment=block.body.blob_kzg_commitments[index],
             kzg_proof=blob_kzg_proofs[index],
+            commitment_inclusion_proof=compute_commitment_inclusion_proof(
+                block.body,
+                get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments', index),
+            ),
+            signed_block_header=signed_block_header,
         )
         for index, blob in enumerate(blobs)
     ]
-
-```
-
-Then for each sidecar, `signed_sidecar = SignedBlobSidecar(message=sidecar, signature=signature)` is constructed and published to the associated sidecar topic, the `blob_sidecar_{subnet_id}` pubsub topic.
-
-`signature` is obtained from:
-
-```python
-def get_blob_sidecar_signature(state: BeaconState,
-                               sidecar: BlobSidecar,
-                               privkey: int) -> BLSSignature:
-    domain = get_domain(state, DOMAIN_BLOB_SIDECAR, compute_epoch_at_slot(sidecar.slot))
-    signing_root = compute_signing_root(sidecar, domain)
-    return bls.Sign(privkey, signing_root)
 ```
 
 The `subnet_id` for the `signed_sidecar` is calculated with:

tests/core/pyspec/eth2spec/test/deneb/unittests/validator/test_validator.py

@@ -12,16 +12,46 @@
 from eth2spec.test.helpers.block import (
     build_empty_block_for_next_slot
 )
-from eth2spec.test.helpers.keys import (
-    pubkey_to_privkey
-)
+from tests.core.pyspec.eth2spec.utils.ssz.ssz_impl import hash_tree_root
+
+
+def get_blob_sidecars(spec, signed_block, blobs, blob_kzg_proofs):
+    block = signed_block.message
+    block_header = spec.BeaconBlockHeader(
+        slot=block.slot,
+        proposer_index=block.proposer_index,
+        parent_root=block.parent_root,
+        state_root=block.state_root,
+        body_root=hash_tree_root(block.body),
+    )
+    signed_block_header = spec.SignedBeaconBlockHeader(message=block_header, signature=signed_block.signature)
+    return [
+        spec.BlobSidecar(
+            index=index,
+            blob=blob,
+            kzg_commitment=signed_block.message.body.blob_kzg_commitments[index],
+            kzg_proof=blob_kzg_proofs[index],
+            commitment_inclusion_proof=compute_commitment_inclusion_proof(
+                spec,
+                signed_block.message.body,
+                index,
+            ),
+            signed_block_header=signed_block_header,
+        )
+        for index, blob in enumerate(blobs)
+    ]
+
+
+def compute_commitment_inclusion_proof(spec, body, index):
+    gindex = spec.get_generalized_index(spec.BeaconBlockBody, 'blob_kzg_commitments', index)
+    return spec.build_proof(body, gindex)
 
 
 @with_deneb_and_later
 @spec_state_test
-def test_blob_sidecar_signature(spec, state):
+def test_blob_sidecar_inclusion_proof(spec, state):
     """
-    Test `get_blob_sidecar_signature`
+    Test `verify_blob_sidecar_inclusion_proof`
     """
     blob_count = 4
     block = build_empty_block_for_next_slot(spec, state)
@@ -30,22 +60,16 @@ def test_blob_sidecar_signature(spec, state):
     block.body.execution_payload.transactions = [opaque_tx]
     block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload)
 
-    blob_sidecars = spec.get_blob_sidecars(block, blobs, proofs)
-    proposer = state.validators[blob_sidecars[1].proposer_index]
-    privkey = pubkey_to_privkey[proposer.pubkey]
-    sidecar_signature = spec.get_blob_sidecar_signature(state,
-                                                        blob_sidecars[1],
-                                                        privkey)
-
-    signed_blob_sidecar = spec.SignedBlobSidecar(message=blob_sidecars[1], signature=sidecar_signature)
+    blob_sidecars = spec.get_blob_sidecars(spec, block, blobs, proofs)
 
-    assert spec.verify_blob_sidecar_signature(state, signed_blob_sidecar)
+    for blob_sidecar in blob_sidecars:
+        assert spec.verify_blob_sidecar_inclusion_proof(blob_sidecar)
 
 
 @with_deneb_and_later
 @spec_state_test
 @always_bls
-def test_blob_sidecar_signature_incorrect(spec, state):
+def test_blob_sidecar_inclusion_proof_incorrect(spec, state):
     """
     Test `get_blob_sidecar_signature`
     """
@@ -56,12 +80,9 @@ def test_blob_sidecar_signature_incorrect(spec, state):
     block.body.execution_payload.transactions = [opaque_tx]
     block.body.execution_payload.block_hash = compute_el_block_hash(spec, block.body.execution_payload)
 
-    blob_sidecars = spec.get_blob_sidecars(block, blobs, proofs)
-
-    sidecar_signature = spec.get_blob_sidecar_signature(state,
-                                                        blob_sidecars[1],
-                                                        123)
-
-    signed_blob_sidecar = spec.SignedBlobSidecar(message=blob_sidecars[1], signature=sidecar_signature)
+    blob_sidecars = spec.get_blob_sidecars(spec, block, blobs, proofs)
 
-    assert not spec.verify_blob_sidecar_signature(state, signed_blob_sidecar)
+    for blob_sidecar in blob_sidecars:
+        block = blob_sidecar.signed_block_header.message
+        block = block.body_root = hash_tree_root(block.body_root)  # mutate body root to break proof
+        assert not spec.verify_blob_sidecar_inclusion_proof(blob_sidecar)