Merge pull request #3531 from dapplion/blob-p2p-proof

Add blob sidecar inclusion proof

presets/mainnet/deneb.yaml

@@ -8,3 +8,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096
 MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096
 # `uint64(6)`
 MAX_BLOBS_PER_BLOCK: 6
+# `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17
+KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17

presets/minimal/deneb.yaml

@@ -8,3 +8,5 @@ FIELD_ELEMENTS_PER_BLOB: 4096
 MAX_BLOB_COMMITMENTS_PER_BLOCK: 16
 # `uint64(6)`
 MAX_BLOBS_PER_BLOCK: 6
+# [customized] `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 4 = 9
+KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9

pysetup/helpers.py

@@ -68,8 +68,7 @@ def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str
         if k in [
             "ceillog2",
             "floorlog2",
-            "compute_merkle_proof_for_block_body",
-            "compute_merkle_proof_for_state",
+            "compute_merkle_proof",
         ]:
             del spec_object.functions[k]
 
@@ -111,8 +110,9 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
         return out
 
     # Merge all constant objects
-    hardcoded_ssz_dep_constants =         reduce(lambda obj, builder: {**obj, **builder.hardcoded_ssz_dep_constants()},                    builders, {})
+    hardcoded_ssz_dep_constants =         reduce(lambda obj, builder: {**obj, **builder.hardcoded_ssz_dep_constants()}, builders, {})
     hardcoded_custom_type_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_custom_type_dep_constants(spec_object)}, builders, {})
+    hardcoded_func_dep_presets = reduce(lambda obj, builder: {**obj, **builder.hardcoded_func_dep_presets(spec_object)}, builders, {})
     # Concatenate all strings
     imports =              reduce(lambda txt, builder: (txt + "\n\n" + builder.imports(preset_name)  ).strip("\n"), builders, "")
     preparations =         reduce(lambda txt, builder: (txt + "\n\n" + builder.preparations()        ).strip("\n"), builders, "")
@@ -126,6 +126,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
     ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
     ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
     custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_custom_type_dep_constants[x]), hardcoded_custom_type_dep_constants))
+    func_dep_presets_verification = '\n'.join(map(lambda x: 'assert %s == %s  # noqa: E501' % (x, spec_object.func_dep_presets[x]), hardcoded_func_dep_presets))
     spec_strs = [
         imports,
         preparations,
@@ -147,6 +148,7 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
         # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are
         # as same as the spec definition.
         ssz_dep_constants_verification,
+        func_dep_presets_verification,
     ]
     return "\n\n\n".join([str.strip("\n") for str in spec_strs if str]) +  "\n"
 
@@ -223,6 +225,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
     preset_vars = combine_dicts(spec0.preset_vars, spec1.preset_vars)
     config_vars = combine_dicts(spec0.config_vars, spec1.config_vars)
     ssz_dep_constants = combine_dicts(spec0.ssz_dep_constants, spec1.ssz_dep_constants)
+    func_dep_presets = combine_dicts(spec0.func_dep_presets, spec1.func_dep_presets)
     ssz_objects = combine_ssz_objects(spec0.ssz_objects, spec1.ssz_objects, custom_types)
     dataclasses = combine_dicts(spec0.dataclasses, spec1.dataclasses)
     return SpecObject(
@@ -233,6 +236,7 @@ def combine_spec_objects(spec0: SpecObject, spec1: SpecObject) -> SpecObject:
         preset_vars=preset_vars,
         config_vars=config_vars,
         ssz_dep_constants=ssz_dep_constants,
+        func_dep_presets=func_dep_presets,
         ssz_objects=ssz_objects,
         dataclasses=dataclasses,
     )

pysetup/spec_builders/altair.py

@@ -27,16 +27,16 @@ def preparations(cls):
     @classmethod
     def sundry_functions(cls) -> str:
         return '''
-def get_generalized_index(ssz_class: Any, *path: Sequence[PyUnion[int, SSZVariableName]]) -> GeneralizedIndex:
+def get_generalized_index(ssz_class: Any, *path: PyUnion[int, SSZVariableName]) -> GeneralizedIndex:
     ssz_path = Path(ssz_class)
     for item in path:
         ssz_path = ssz_path / item
     return GeneralizedIndex(ssz_path.gindex())
 
 
-def compute_merkle_proof_for_state(state: BeaconState,
-                                   index: GeneralizedIndex) -> Sequence[Bytes32]:
-    return build_proof(state.get_backing(), index)'''
+def compute_merkle_proof(object: SSZObject,
+                         index: GeneralizedIndex) -> Sequence[Bytes32]:
+    return build_proof(object.get_backing(), index)'''
 
 
     @classmethod

pysetup/spec_builders/base.py

@@ -47,6 +47,10 @@ def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:  #
         """
         return {}
 
+    @classmethod
+    def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]:
+        return {}
+
     @classmethod
     def implement_optimizations(cls, functions: Dict[str, str]) -> Dict[str, str]:
         return functions

pysetup/spec_builders/capella.py

@@ -13,15 +13,6 @@ def imports(cls, preset_name: str):
 from eth2spec.bellatrix import {preset_name} as bellatrix
 '''
 
-
-    @classmethod
-    def sundry_functions(cls) -> str:
-        return '''
-def compute_merkle_proof_for_block_body(body: BeaconBlockBody,
-                                        index: GeneralizedIndex) -> Sequence[Bytes32]:
-    return build_proof(body.get_backing(), index)'''
-
-
     @classmethod
     def hardcoded_ssz_dep_constants(cls) -> Dict[str, str]:
         return {

pysetup/spec_builders/deneb.py

@@ -1,3 +1,4 @@
+from typing import Dict
 from .base import BaseSpecBuilder
 from ..constants import DENEB
 
@@ -23,7 +24,8 @@ def sundry_functions(cls) -> str:
         return '''
 def retrieve_blobs_and_proofs(beacon_block_root: Root) -> Tuple[Sequence[Blob], Sequence[KZGProof]]:
     # pylint: disable=unused-argument
-    return [], []'''
+    return [], []
+'''
 
     @classmethod
     def execution_engine_cls(cls) -> str:
@@ -63,9 +65,16 @@ def verify_and_notify_new_payload(self: ExecutionEngine,
 
 
     @classmethod
-    def hardcoded_custom_type_dep_constants(cls, spec_object) -> str:
+    def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:
         return {
             'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value,
             'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value,
             'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value,
+            'MAX_BLOB_COMMITMENTS_PER_BLOCK': spec_object.preset_vars['MAX_BLOB_COMMITMENTS_PER_BLOCK'].value,
+        }
+
+    @classmethod
+    def hardcoded_func_dep_presets(cls, spec_object) -> Dict[str, str]:
+        return {
+            'KZG_COMMITMENT_INCLUSION_PROOF_DEPTH': spec_object.preset_vars['KZG_COMMITMENT_INCLUSION_PROOF_DEPTH'].value,
         }

pysetup/typing.py

@@ -22,6 +22,7 @@ class SpecObject(NamedTuple):
     preset_vars: Dict[str, VariableDefinition]
     config_vars: Dict[str, VariableDefinition]
     ssz_dep_constants: Dict[str, str]  # the constants that depend on ssz_objects
+    func_dep_presets: Dict[str, str]  # the constants that depend on functions
     ssz_objects: Dict[str, str]
     dataclasses: Dict[str, str]
 

setup.py

@@ -162,6 +162,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
     preset_vars: Dict[str, VariableDefinition] = {}
     config_vars: Dict[str, VariableDefinition] = {}
     ssz_dep_constants: Dict[str, str] = {}
+    func_dep_presets: Dict[str, str] = {}
     ssz_objects: Dict[str, str] = {}
     dataclasses: Dict[str, str] = {}
     custom_types: Dict[str, str] = {}
@@ -214,6 +215,16 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
 
                     value_cell = cells[1]
                     value = value_cell.children[0].children
+
+                    description = None
+                    if len(cells) >= 3:
+                        description_cell = cells[2]
+                        if len(description_cell.children) > 0:
+                            description = description_cell.children[0].children
+                            if isinstance(description, list):
+                                # marko parses `**X**` as a list containing a X
+                                description = description[0].children
+
                     if isinstance(value, list):
                         # marko parses `**X**` as a list containing a X
                         value = value[0].children
@@ -228,6 +239,9 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
                         ssz_dep_constants[name] = value
                         continue
 
+                    if description is not None and description.startswith("<!-- predefined -->"):
+                        func_dep_presets[name] = value
+
                     value_def = _parse_value(name, value)
                     if name in preset:
                         preset_vars[name] = VariableDefinition(value_def.type_name, preset[name], value_def.comment, None)
@@ -256,6 +270,7 @@ def get_spec(file_name: Path, preset: Dict[str, str], config: Dict[str, str], pr
         preset_vars=preset_vars,
         config_vars=config_vars,
         ssz_dep_constants=ssz_dep_constants,
+        func_dep_presets=func_dep_presets,
         ssz_objects=ssz_objects,
         dataclasses=dataclasses,
     )

specs/altair/light-client/full-node.md

@@ -10,7 +10,7 @@
 
 - [Introduction](#introduction)
 - [Helper functions](#helper-functions)
-  - [`compute_merkle_proof_for_state`](#compute_merkle_proof_for_state)
+  - [`compute_merkle_proof`](#compute_merkle_proof)
   - [`block_to_light_client_header`](#block_to_light_client_header)
 - [Deriving light client data](#deriving-light-client-data)
   - [`create_light_client_bootstrap`](#create_light_client_bootstrap)
@@ -27,11 +27,13 @@ This document provides helper functions to enable full nodes to serve light clie
 
 ## Helper functions
 
-### `compute_merkle_proof_for_state`
+### `compute_merkle_proof`
+
+This function return the Merkle proof of the given SSZ object `object` at generalized index `index`.
 
 ```python
-def compute_merkle_proof_for_state(state: BeaconState,
-                                   index: GeneralizedIndex) -> Sequence[Bytes32]:
+def compute_merkle_proof(object: SSZObject,
+                         index: GeneralizedIndex) -> Sequence[Bytes32]:
     ...
 ```
 
@@ -73,7 +75,7 @@ def create_light_client_bootstrap(state: BeaconState,
     return LightClientBootstrap(
         header=block_to_light_client_header(block),
         current_sync_committee=state.current_sync_committee,
-        current_sync_committee_branch=compute_merkle_proof_for_state(state, CURRENT_SYNC_COMMITTEE_INDEX),
+        current_sync_committee_branch=compute_merkle_proof(state, CURRENT_SYNC_COMMITTEE_INDEX),
     )
 ```
 
@@ -120,8 +122,7 @@ def create_light_client_update(state: BeaconState,
     # `next_sync_committee` is only useful if the message is signed by the current sync committee
     if update_attested_period == update_signature_period:
         update.next_sync_committee = attested_state.next_sync_committee
-        update.next_sync_committee_branch = compute_merkle_proof_for_state(
-            attested_state, NEXT_SYNC_COMMITTEE_INDEX)
+        update.next_sync_committee_branch = compute_merkle_proof(attested_state, NEXT_SYNC_COMMITTEE_INDEX)
 
     # Indicate finality whenever possible
     if finalized_block is not None:
@@ -130,8 +131,7 @@ def create_light_client_update(state: BeaconState,
             assert hash_tree_root(update.finalized_header.beacon) == attested_state.finalized_checkpoint.root
         else:
             assert attested_state.finalized_checkpoint.root == Bytes32()
-        update.finality_branch = compute_merkle_proof_for_state(
-            attested_state, FINALIZED_ROOT_INDEX)
+        update.finality_branch = compute_merkle_proof(attested_state, FINALIZED_ROOT_INDEX)
 
     update.sync_aggregate = block.message.body.sync_aggregate
     update.signature_slot = block.message.slot

specs/capella/light-client/full-node.md

@@ -10,7 +10,6 @@
 
 - [Introduction](#introduction)
 - [Helper functions](#helper-functions)
-  - [`compute_merkle_proof_for_block_body`](#compute_merkle_proof_for_block_body)
   - [Modified `block_to_light_client_header`](#modified-block_to_light_client_header)
 
 <!-- END doctoc generated TOC please keep comment here to allow auto update -->
@@ -22,14 +21,6 @@ This upgrade adds information about the execution payload to light client data a
 
 ## Helper functions
 
-### `compute_merkle_proof_for_block_body`
-
-```python
-def compute_merkle_proof_for_block_body(body: BeaconBlockBody,
-                                        index: GeneralizedIndex) -> Sequence[Bytes32]:
-    ...
-```
-
 ### Modified `block_to_light_client_header`
 
 ```python
@@ -55,7 +46,7 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader:
             transactions_root=hash_tree_root(payload.transactions),
             withdrawals_root=hash_tree_root(payload.withdrawals),
         )
-        execution_branch = compute_merkle_proof_for_block_body(block.message.body, EXECUTION_PAYLOAD_INDEX)
+        execution_branch = compute_merkle_proof(block.message.body, EXECUTION_PAYLOAD_INDEX)
     else:
         # Note that during fork transitions, `finalized_header` may still point to earlier forks.
         # While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`),

specs/deneb/light-client/full-node.md

@@ -52,7 +52,7 @@ def block_to_light_client_header(block: SignedBeaconBlock) -> LightClientHeader:
             execution_header.blob_gas_used = payload.blob_gas_used
             execution_header.excess_blob_gas = payload.excess_blob_gas
 
-        execution_branch = compute_merkle_proof_for_block_body(block.message.body, EXECUTION_PAYLOAD_INDEX)
+        execution_branch = compute_merkle_proof(block.message.body, EXECUTION_PAYLOAD_INDEX)
     else:
         # Note that during fork transitions, `finalized_header` may still point to earlier forks.
         # While Bellatrix blocks also contain an `ExecutionPayload` (minus `withdrawals_root`),