Merge branch 'dev' into lc-electra

configs/mainnet.yaml

@@ -147,6 +147,8 @@ MAX_REQUEST_BLOB_SIDECARS: 768
 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096
 # `6`
 BLOB_SIDECAR_SUBNET_COUNT: 6
+## `uint64(6)`
+MAX_BLOBS_PER_BLOCK: 6
 
 # Whisk
 # `Epoch(2**8)`

configs/minimal.yaml

@@ -148,6 +148,8 @@ MAX_REQUEST_BLOB_SIDECARS: 768
 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096
 # `6`
 BLOB_SIDECAR_SUBNET_COUNT: 6
+## `uint64(6)`
+MAX_BLOBS_PER_BLOCK: 6
 
 # Whisk
 WHISK_EPOCHS_PER_SHUFFLING_PHASE: 4

presets/mainnet/deneb.yaml

@@ -6,7 +6,5 @@
 FIELD_ELEMENTS_PER_BLOB: 4096
 # `uint64(2**12)` (= 4096)
 MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096
-# `uint64(6)`
-MAX_BLOBS_PER_BLOCK: 6
 # `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 12 = 17
 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 17

presets/minimal/deneb.yaml

@@ -6,7 +6,5 @@
 FIELD_ELEMENTS_PER_BLOB: 4096
 # [customized]
 MAX_BLOB_COMMITMENTS_PER_BLOCK: 16
-# `uint64(6)`
-MAX_BLOBS_PER_BLOCK: 6
 # [customized] `floorlog2(get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments')) + 1 + ceillog2(MAX_BLOB_COMMITMENTS_PER_BLOCK)` = 4 + 1 + 4 = 9
 KZG_COMMITMENT_INCLUSION_PROOF_DEPTH: 9

pysetup/spec_builders/deneb.py

@@ -70,7 +70,7 @@ def hardcoded_custom_type_dep_constants(cls, spec_object) -> Dict[str, str]:
  return {
  'BYTES_PER_FIELD_ELEMENT': spec_object.constant_vars['BYTES_PER_FIELD_ELEMENT'].value,
  'FIELD_ELEMENTS_PER_BLOB': spec_object.preset_vars['FIELD_ELEMENTS_PER_BLOB'].value,
- 'MAX_BLOBS_PER_BLOCK': spec_object.preset_vars['MAX_BLOBS_PER_BLOCK'].value,
+ 'MAX_BLOBS_PER_BLOCK': spec_object.config_vars['MAX_BLOBS_PER_BLOCK'].value,
  'MAX_BLOB_COMMITMENTS_PER_BLOCK': spec_object.preset_vars['MAX_BLOB_COMMITMENTS_PER_BLOCK'].value,
  }
 

specs/_features/eip7594/das-core.md

@@ -23,6 +23,7 @@
  - [`compute_extended_matrix`](#compute_extended_matrix)
  - [`recover_matrix`](#recover_matrix)
  - [`get_data_column_sidecars`](#get_data_column_sidecars)
+ - [`get_extended_sample_count`](#get_extended_sample_count)
 - [Custody](#custody)
  - [Custody requirement](#custody-requirement)
  - [Public, deterministic selection](#public-deterministic-selection)
@@ -31,6 +32,8 @@
 - [Column gossip](#column-gossip)
  - [Parameters](#parameters)
 - [Peer sampling](#peer-sampling)
+ - [Sample selection](#sample-selection)
+ - [Sample queries](#sample-queries)
 - [Peer scoring](#peer-scoring)
 - [Reconstruction and cross-seeding](#reconstruction-and-cross-seeding)
 - [DAS providers](#das-providers)
@@ -65,8 +68,8 @@ The following values are (non-configurable) constants used throughout the specif
 
 | Name | Value | Description |
 | - | - | - |
-| `NUMBER_OF_COLUMNS` | `uint64(CELLS_PER_EXT_BLOB)` (= 128) | Number of columns in the extended data matrix. |
-| `MAX_CELLS_IN_EXTENDED_MATRIX` | `uint64(MAX_BLOBS_PER_BLOCK * NUMBER_OF_COLUMNS)` (= 768) | The data size of `ExtendedMatrix`. |
+| `NUMBER_OF_COLUMNS` | `uint64(CELLS_PER_EXT_BLOB)` (= 128) | Number of columns in the extended data matrix |
+| `MAX_CELLS_IN_EXTENDED_MATRIX` | `uint64(MAX_BLOBS_PER_BLOCK * NUMBER_OF_COLUMNS)` (= 768) | The data size of `ExtendedMatrix` |
 
 ### Networking
 
@@ -176,9 +179,8 @@ def recover_matrix(partial_matrix: Sequence[MatrixEntry],
  for blob_index in range(blob_count):
  cell_indices = [e.column_index for e in partial_matrix if e.row_index == blob_index]
  cells = [e.cell for e in partial_matrix if e.row_index == blob_index]
- proofs = [e.kzg_proof for e in partial_matrix if e.row_index == blob_index]
 
- recovered_cells, recovered_proofs = recover_cells_and_kzg_proofs(cell_indices, cells, proofs)
+ recovered_cells, recovered_proofs = recover_cells_and_kzg_proofs(cell_indices, cells)
  for cell_index, (cell, proof) in enumerate(zip(recovered_cells, recovered_proofs)):
  extended_matrix.append(MatrixEntry(
  cell=cell,
@@ -193,34 +195,80 @@ def recover_matrix(partial_matrix: Sequence[MatrixEntry],
 
 ```python
 def get_data_column_sidecars(signed_block: SignedBeaconBlock,
- blobs: Sequence[Blob]) -> Sequence[DataColumnSidecar]:
+ cells_and_kzg_proofs: Sequence[Tuple[
+ Vector[Cell, CELLS_PER_EXT_BLOB],
+ Vector[KZGProof, CELLS_PER_EXT_BLOB]]]) -> Sequence[DataColumnSidecar]:
+ """
+ Given a signed block and the cells/proofs associated with each blob in the
+ block, assemble the sidecars which can be distributed to peers.
+ """
+ blob_kzg_commitments = signed_block.message.body.blob_kzg_commitments
+ assert len(cells_and_kzg_proofs) == len(blob_kzg_commitments)
  signed_block_header = compute_signed_block_header(signed_block)
- block = signed_block.message
  kzg_commitments_inclusion_proof = compute_merkle_proof(
- block.body,
+ signed_block.message.body,
  get_generalized_index(BeaconBlockBody, 'blob_kzg_commitments'),
  )
- cells_and_proofs = [compute_cells_and_kzg_proofs(blob) for blob in blobs]
- blob_count = len(blobs)
- cells = [cells_and_proofs[i][0] for i in range(blob_count)]
- proofs = [cells_and_proofs[i][1] for i in range(blob_count)]
+
  sidecars = []
  for column_index in range(NUMBER_OF_COLUMNS):
- column_cells = [cells[row_index][column_index]
-  for row_index in range(blob_count)]
- column_proofs = [proofs[row_index][column_index]
-  for row_index in range(blob_count)]
+ column_cells, column_proofs = [], []
+ for cells, proofs in cells_and_kzg_proofs:
+  column_cells.append(cells[column_index])
+ column_proofs.append(proofs[column_index])
  sidecars.append(DataColumnSidecar(
  index=column_index,
  column=column_cells,
- kzg_commitments=block.body.blob_kzg_commitments,
+ kzg_commitments=blob_kzg_commitments,
  kzg_proofs=column_proofs,
  signed_block_header=signed_block_header,
  kzg_commitments_inclusion_proof=kzg_commitments_inclusion_proof,
  ))
  return sidecars
 ```
 
+#### `get_extended_sample_count`
+
+```python
+def get_extended_sample_count(allowed_failures: uint64) -> uint64:
+ assert 0 <= allowed_failures <= NUMBER_OF_COLUMNS // 2
+ """
+ Return the sample count if allowing failures.
+
+ This helper demonstrates how to calculate the number of columns to query per slot when
+ allowing given number of failures, assuming uniform random selection without replacement.
+ Nested functions are direct replacements of Python library functions math.comb and
+ scipy.stats.hypergeom.cdf, with the same signatures.
+ """
+
+ def math_comb(n: int, k: int) -> int:
+ if not 0 <= k <= n:
+ return 0
+ r = 1
+ for i in range(min(k, n - k)):
+ r = r * (n - i) // (i + 1)
+ return r
+
+ def hypergeom_cdf(k: uint64, M: uint64, n: uint64, N: uint64) -> float:
+ # NOTE: It contains float-point computations.
+ # Convert uint64 to Python integers before computations.
+ k = int(k)
+ M = int(M)
+ n = int(n)
+ N = int(N)
+ return sum([math_comb(n, i) * math_comb(M - n, N - i) / math_comb(M, N)
+ for i in range(k + 1)])
+
+ worst_case_missing = NUMBER_OF_COLUMNS // 2 + 1
+ false_positive_threshold = hypergeom_cdf(0, NUMBER_OF_COLUMNS,
+ worst_case_missing, SAMPLES_PER_SLOT)
+ for sample_count in range(SAMPLES_PER_SLOT, NUMBER_OF_COLUMNS + 1):
+ if hypergeom_cdf(allowed_failures, NUMBER_OF_COLUMNS,
+ worst_case_missing, sample_count) <= false_positive_threshold:
+ break
+ return sample_count
+```
+
 ## Custody
 
 ### Custody requirement
@@ -259,23 +307,43 @@ In this construction, we extend the blobs using a one-dimensional erasure coding
 
 For each column -- use `data_column_sidecar_{subnet_id}` subnets, where `subnet_id` can be computed with the `compute_subnet_for_data_column_sidecar(column_index: ColumnIndex)` helper. The sidecars can be computed with the `get_data_column_sidecars(signed_block: SignedBeaconBlock, blobs: Sequence[Blob])` helper.
 
-To custody a particular column, a node joins the respective gossip subnet. Verifiable samples from their respective column are gossiped on the assigned subnet.
+Verifiable samples from their respective column are distributed on the assigned subnet. To custody a particular column, a node joins the respective gossipsub subnet. If a node fails to get a column on the column subnet, a node can also utilize the Req/Resp protocol to query the missing column from other peers.
 
 ## Peer sampling
 
-A node SHOULD maintain a diverse set of peers for each column and each slot by verifying responsiveness to sample queries. At each slot, a node makes `SAMPLES_PER_SLOT` queries for samples from their peers via `DataColumnSidecarsByRoot` request. A node utilizes `get_custody_columns` helper to determine which peer(s) to request from. If a node has enough good/honest peers across all rows and columns, this has a high chance of success.
+### Sample selection
+
+At each slot, a node SHOULD select at least `SAMPLES_PER_SLOT` column IDs for sampling. It is recommended to use uniform random selection without replacement based on local randomness. Sampling is considered successful if the node manages to retrieve all selected columns.
+
+Alternatively, a node MAY use a method that selects more than `SAMPLES_PER_SLOT` columns while allowing some missing, respecting the same target false positive threshold (the probability of successful sampling of an unavailable block) as dictated by the `SAMPLES_PER_SLOT` parameter. If using uniform random selection without replacement, a node can use the `get_extended_sample_count(allowed_failures) -> sample_count` helper function to determine the sample count (number of unique column IDs) for any selected number of allowed failures. Sampling is then considered successful if any `sample_count - allowed_failures` columns are retrieved successfully.
+
+For reference, the table below shows the number of samples and the number of allowed missing columns assuming `NUMBER_OF_COLUMNS = 128` and `SAMPLES_PER_SLOT = 16`.
+
+| Allowed missing | 0| 1| 2| 3| 4| 5| 6| 7| 8|
+|-----------------|--|--|--|--|--|--|--|--|--|
+| Sample count |16|20|24|27|29|32|35|37|40|
+
+### Sample queries
+
+A node SHOULD maintain a diverse set of peers for each column and each slot by verifying responsiveness to sample queries.
+
+A node SHOULD query for samples from selected peers via `DataColumnSidecarsByRoot` request. A node utilizes `get_custody_columns` helper to determine which peer(s) it could request from, identifying a list of candidate peers for each selected column.
+
+If more than one candidate peer is found for a given column, a node SHOULD randomize its peer selection to distribute sample query load in the network. Nodes MAY use peer scoring to tune this selection (for example, by using weighted selection or by using a cut-off threshold). If possible, it is also recommended to avoid requesting many columns from the same peer in order to avoid relying on and exposing the sample selection to a single peer.
+
+If a node already has a column because of custody, it is not required to send out queries for that column.
+
+If a node has enough good/honest peers across all columns, and the data is being made available, the above procedure has a high chance of success.
 
 ## Peer scoring
 
 Due to the deterministic custody functions, a node knows exactly what a peer should be able to respond to. In the event that a peer does not respond to samples of their custodied rows/columns, a node may downscore or disconnect from a peer.
 
 ## Reconstruction and cross-seeding
 
-If the node obtains 50%+ of all the columns, they can reconstruct the full data matrix via `recover_matrix` helper.
-
-If a node fails to sample a peer or fails to get a column on the column subnet, a node can utilize the Req/Resp message to query the missing column from other peers.
+If the node obtains 50%+ of all the columns, it SHOULD reconstruct the full data matrix via `recover_matrix` helper. Nodes MAY delay this reconstruction allowing time for other columns to arrive over the network. If delaying reconstruction, nodes may use a random delay in order to desynchronize reconstruction among nodes, thus reducing overall CPU load.
 
-Once the node obtain the column, the node SHOULD send the missing columns to the column subnets.
+Once the node obtains a column through reconstruction, the node MUST expose the new column as if it had received it over the network. If the node is subscribed to the subnet corresponding to the column, it MUST send the reconstructed DataColumnSidecar to its topic mesh neighbors. If instead the node is not subscribed to the corresponding subnet, it SHOULD still expose the availability of the DataColumnSidecar as part of the gossip emission process.
 
 *Note*: A node always maintains a matrix view of the rows and columns they are following, able to cross-reference and cross-seed in either direction.
 

specs/_features/eip7594/p2p-interface.md

@@ -68,19 +68,18 @@ class DataColumnIdentifier(Container):
 ```python
 def verify_data_column_sidecar_kzg_proofs(sidecar: DataColumnSidecar) -> bool:
  """
- Verify if the proofs are correct
+ Verify if the proofs are correct.
  """
  assert sidecar.index < NUMBER_OF_COLUMNS
  assert len(sidecar.column) == len(sidecar.kzg_commitments) == len(sidecar.kzg_proofs)
 
- row_indices = [RowIndex(i) for i in range(len(sidecar.column))]
- column_indices = [sidecar.index] * len(sidecar.column)
+ # The column index also represents the cell index
+ cell_indices = [CellIndex(sidecar.index)] * len(sidecar.column)
 
- # KZG batch verifies that the cells match the corresponding commitments and proofs
+ # Batch verify that the cells match the corresponding commitments and proofs
  return verify_cell_kzg_proof_batch(
- row_commitments_bytes=sidecar.kzg_commitments,
- row_indices=row_indices, # all rows
- column_indices=column_indices, # specific column
+ commitments_bytes=sidecar.kzg_commitments,
+ cell_indices=cell_indices,
  cells=sidecar.column,
  proofs_bytes=sidecar.kzg_proofs,
  )