Merge branch 'dev' into refactor-setup-py

Makefile

@@ -146,8 +146,8 @@ codespell:
 lint: pyspec
  . venv/bin/activate; cd $(PY_SPEC_DIR); \
  flake8 --config $(LINTER_CONFIG_FILE) ./eth2spec \
- && pylint --rcfile $(LINTER_CONFIG_FILE) $(PYLINT_SCOPE) \
- && mypy --config-file $(LINTER_CONFIG_FILE) $(MYPY_SCOPE)
+ && python -m pylint --rcfile $(LINTER_CONFIG_FILE) $(PYLINT_SCOPE) \
+ && python -m mypy --config-file $(LINTER_CONFIG_FILE) $(MYPY_SCOPE)
 
 lint_generators: pyspec
  . venv/bin/activate; cd $(TEST_GENERATORS_DIR); \

configs/mainnet.yaml

@@ -98,16 +98,16 @@ DEPOSIT_CONTRACT_ADDRESS: 0x00000000219ab540356cBB839Cbe05303d7705Fa
 
 # Networking
 # ---------------------------------------------------------------
-# `2**20` (= 1048576, 1 MiB)
-GOSSIP_MAX_SIZE: 1048576
+# `10 * 2**20` (= 10485760, 10 MiB)
+GOSSIP_MAX_SIZE: 10485760
 # `2**10` (= 1024)
 MAX_REQUEST_BLOCKS: 1024
 # `2**8` (= 256)
 EPOCHS_PER_SUBNET_SUBSCRIPTION: 256
 # `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 33024, ~5 months)
 MIN_EPOCHS_FOR_BLOCK_REQUESTS: 33024
-# `2**20` (=1048576, 1 MiB) 
-MAX_CHUNK_SIZE: 1048576
+# `10 * 2**20` (=10485760, 10 MiB)
+MAX_CHUNK_SIZE: 10485760
 # 5s
 TTFB_TIMEOUT: 5
 # 10s
@@ -124,3 +124,11 @@ ATTESTATION_SUBNET_COUNT: 64
 ATTESTATION_SUBNET_EXTRA_BITS: 0
 # ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS
 ATTESTATION_SUBNET_PREFIX_BITS: 6
+
+# Deneb
+# `2**7` (=128)
+MAX_REQUEST_BLOCKS_DENEB: 128
+# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK
+MAX_REQUEST_BLOB_SIDECARS: 768
+# `2**12` (= 4096 epochs, ~18 days)
+MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096

configs/minimal.yaml

@@ -99,16 +99,16 @@ DEPOSIT_CONTRACT_ADDRESS: 0x1234567890123456789012345678901234567890
 
 # Networking
 # ---------------------------------------------------------------
-# `2**20` (= 1048576, 1 MiB)
-GOSSIP_MAX_SIZE: 1048576
+# `10 * 2**20` (= 10485760, 10 MiB)
+GOSSIP_MAX_SIZE: 10485760
 # `2**10` (= 1024)
 MAX_REQUEST_BLOCKS: 1024
 # `2**8` (= 256)
 EPOCHS_PER_SUBNET_SUBSCRIPTION: 256
 # [customized] `MIN_VALIDATOR_WITHDRAWABILITY_DELAY + CHURN_LIMIT_QUOTIENT // 2` (= 272)
 MIN_EPOCHS_FOR_BLOCK_REQUESTS: 272
-# `2**20` (=1048576, 1 MiB) 
-MAX_CHUNK_SIZE: 1048576
+# `10 * 2**20` (=10485760, 10 MiB)
+MAX_CHUNK_SIZE: 10485760
 # 5s
 TTFB_TIMEOUT: 5
 # 10s
@@ -125,3 +125,11 @@ ATTESTATION_SUBNET_COUNT: 64
 ATTESTATION_SUBNET_EXTRA_BITS: 0
 # ceillog2(ATTESTATION_SUBNET_COUNT) + ATTESTATION_SUBNET_EXTRA_BITS
 ATTESTATION_SUBNET_PREFIX_BITS: 6
+
+# Deneb
+# `2**7` (=128)
+MAX_REQUEST_BLOCKS_DENEB: 128
+# MAX_REQUEST_BLOCKS_DENEB * MAX_BLOBS_PER_BLOCK
+MAX_REQUEST_BLOB_SIDECARS: 768
+# `2**12` (= 4096 epochs, ~18 days)
+MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096

presets/mainnet/deneb.yaml

@@ -6,5 +6,5 @@
 FIELD_ELEMENTS_PER_BLOB: 4096
 # `uint64(2**12)` (= 4096)
 MAX_BLOB_COMMITMENTS_PER_BLOCK: 4096
-# `uint64(2**2)` (= 4)
-MAX_BLOBS_PER_BLOCK: 4
+# `uint64(6)`
+MAX_BLOBS_PER_BLOCK: 6

presets/mainnet/phase0.yaml

@@ -85,4 +85,4 @@ MAX_ATTESTATIONS: 128
 # 2**4 (= 16)
 MAX_DEPOSITS: 16
 # 2**4 (= 16)
-MAX_VOLUNTARY_EXITS: 16
+MAX_VOLUNTARY_EXITS: 16

presets/minimal/deneb.yaml

@@ -6,5 +6,5 @@
 FIELD_ELEMENTS_PER_BLOB: 4
 # [customized]
 MAX_BLOB_COMMITMENTS_PER_BLOCK: 16
-# `uint64(2**2)` (= 4)
-MAX_BLOBS_PER_BLOCK: 4
+# `uint64(6)`
+MAX_BLOBS_PER_BLOCK: 6

presets/minimal/phase0.yaml

@@ -85,4 +85,4 @@ MAX_ATTESTATIONS: 128
 # 2**4 (= 16)
 MAX_DEPOSITS: 16
 # 2**4 (= 16)
-MAX_VOLUNTARY_EXITS: 16
+MAX_VOLUNTARY_EXITS: 16

pysetup/md_doc_paths.py

@@ -1,3 +1,5 @@
+import os
+
 from .constants import (
  PHASE0,
  ALTAIR,
@@ -9,70 +11,65 @@
 )
 
 
+PREVIOUS_FORK_OF = {
+ PHASE0: None,
+ ALTAIR: PHASE0,
+ BELLATRIX: ALTAIR,
+ CAPELLA: BELLATRIX,
+ DENEB: CAPELLA,
+ EIP6110: DENEB,
+ WHISK: CAPELLA,
+}
+
+ALL_FORKS = list(PREVIOUS_FORK_OF.keys())
+
+IGNORE_SPEC_FILES = [
+ "specs/phase0/deposit-contract.md"
+]
+
+EXTRA_SPEC_FILES = {
+ BELLATRIX: "sync/optimistic.md"
+}
+
+
+def is_post_fork(a, b) -> bool:
+ """
+ Returns true if fork a is after b, or if a == b
+ """
+ if a == b:
+ return True
+
+ prev_fork = PREVIOUS_FORK_OF[a]
+ if prev_fork == b:
+ return True
+ elif prev_fork == None:
+ return False
+ else:
+ return is_post_fork(prev_fork, b)
+
+def get_fork_directory(fork):
+ dir1 = f'specs/{fork}'
+ if os.path.exists(dir1):
+ return dir1
+ dir2 = f'specs/_features/{fork}'
+ if os.path.exists(dir2):
+ return dir2
+ raise FileNotFoundError(f"No directory found for fork: {fork}")
+
+
 def get_md_doc_paths(spec_fork: str) -> str:
  md_doc_paths = ""
- if spec_fork in (PHASE0, ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK):
- md_doc_paths += """
- specs/phase0/beacon-chain.md
- specs/phase0/fork-choice.md
- specs/phase0/validator.md
- specs/phase0/weak-subjectivity.md
- specs/phase0/p2p-interface.md
- """
- if spec_fork in (ALTAIR, BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK):
- md_doc_paths += """
- specs/altair/light-client/full-node.md
- specs/altair/light-client/light-client.md
- specs/altair/light-client/p2p-interface.md
- specs/altair/light-client/sync-protocol.md
- specs/altair/beacon-chain.md
- specs/altair/bls.md
- specs/altair/fork.md
- specs/altair/validator.md
- specs/altair/p2p-interface.md
- """
- if spec_fork in (BELLATRIX, CAPELLA, DENEB, EIP6110, WHISK):
- md_doc_paths += """
- specs/bellatrix/beacon-chain.md
- specs/bellatrix/fork.md
- specs/bellatrix/fork-choice.md
- specs/bellatrix/validator.md
- specs/bellatrix/p2p-interface.md
- sync/optimistic.md
- """
- if spec_fork in (CAPELLA, DENEB, EIP6110, WHISK):
- md_doc_paths += """
- specs/capella/light-client/fork.md
- specs/capella/light-client/full-node.md
- specs/capella/light-client/p2p-interface.md
- specs/capella/light-client/sync-protocol.md
- specs/capella/beacon-chain.md
- specs/capella/fork.md
- specs/capella/fork-choice.md
- specs/capella/validator.md
- specs/capella/p2p-interface.md
- """
- if spec_fork in (DENEB, EIP6110):
- md_doc_paths += """
- specs/deneb/light-client/fork.md
- specs/deneb/light-client/full-node.md
- specs/deneb/light-client/p2p-interface.md
- specs/deneb/light-client/sync-protocol.md
- specs/deneb/beacon-chain.md
- specs/deneb/fork.md
- specs/deneb/fork-choice.md
- specs/deneb/polynomial-commitments.md
- specs/deneb/p2p-interface.md
- specs/deneb/validator.md
- """
- if spec_fork == EIP6110:
- md_doc_paths += """
- specs/_features/eip6110/beacon-chain.md
- specs/_features/eip6110/fork.md
- """
- if spec_fork == WHISK:
- md_doc_paths += """
- specs/_features/whisk/beacon-chain.md
- specs/_features/whisk/fork.md
- """
+
+ for fork in ALL_FORKS:
+ if is_post_fork(spec_fork, fork):
+ # Append all files in fork directory recursively
+ for root, dirs, files in os.walk(get_fork_directory(fork)):
+ for filename in files:
+ filepath = os.path.join(root, filename)
+ if filepath.endswith('.md') and filepath not in IGNORE_SPEC_FILES:
+ md_doc_paths += filepath + "\n"
+ # Append extra files if any
+ if fork in EXTRA_SPEC_FILES:
+ md_doc_paths += EXTRA_SPEC_FILES[fork] + "\n"
+
  return md_doc_paths

setup.py

@@ -515,7 +515,7 @@ def run(self):
  "remerkleable==0.1.27",
  "trie==2.0.2",
  RUAMEL_YAML_VERSION,
- "lru-dict==1.1.8",
+ "lru-dict==1.2.0",
  MARKO_VERSION,
  "py_arkworks_bls12381==0.3.4",
  "curdleproofs @ git+https://github.com/nalinbhardwaj/curdleproofs.pie@master#egg=curdleproofs&subdirectory=curdleproofs",

specs/_features/whisk/beacon-chain.md

@@ -19,7 +19,6 @@
  - [`BeaconState`](#beaconstate)
 - [Block processing](#block-processing)
  - [Block header](#block-header)
- - [`BeaconBlock`](#beaconblock)
  - [Whisk](#whisk)
  - [`BeaconBlockBody`](#beaconblockbody)
  - [Deposits](#deposits)
@@ -235,23 +234,11 @@ def process_epoch(state: BeaconState) -> None:
 
 ### Block header
 
-#### `BeaconBlock`
-
-```python
-class BeaconBlock(Container):
- slot: Slot
- proposer_index: ValidatorIndex
- parent_root: Root
- state_root: Root
- body: BeaconBlockBody
- whisk_opening_proof: WhiskTrackerProof # [New in Whisk]
-```
-
 ```python
 def process_whisk_opening_proof(state: BeaconState, block: BeaconBlock) -> None:
  tracker = state.whisk_proposer_trackers[state.slot % WHISK_PROPOSER_TRACKERS_COUNT]
  k_commitment = state.validators[block.proposer_index].whisk_k_commitment
- assert IsValidWhiskOpeningProof(tracker, k_commitment, block.whisk_opening_proof)
+ assert IsValidWhiskOpeningProof(tracker, k_commitment, block.body.whisk_opening_proof)
 ```
 
 Removed `assert block.proposer_index == get_beacon_proposer_index(state)` check in Whisk.
@@ -303,6 +290,8 @@ class BeaconBlockBody(capella.BeaconBlockBody):
  execution_payload: ExecutionPayload
  # Capella operations
  bls_to_execution_changes: List[SignedBLSToExecutionChange, MAX_BLS_TO_EXECUTION_CHANGES]
+ # Whisk
+ whisk_opening_proof: WhiskTrackerProof # [New in Whisk]
  whisk_post_shuffle_trackers: Vector[WhiskTracker, WHISK_VALIDATORS_PER_SHUFFLE] # [New in Whisk]
  whisk_shuffle_proof: WhiskShuffleProof # [New in Whisk]
  whisk_shuffle_proof_M_commitment: BLSG1Point # [New in Whisk]

specs/bellatrix/p2p-interface.md

@@ -14,7 +14,6 @@ Readers should understand the Phase 0 and Altair documents and use them as a bas
 
  - [Warning](#warning)
  - [Modifications in Bellatrix](#modifications-in-bellatrix)
- - [Configuration](#configuration)
  - [The gossip domain: gossipsub](#the-gossip-domain-gossipsub)
  - [Topics and messages](#topics-and-messages)
  - [Global topics](#global-topics)
@@ -41,15 +40,6 @@ Refer to the note in the [validator guide](./validator.md) for further details.
 
 ## Modifications in Bellatrix
 
-### Configuration
-
-This section outlines modifications constants that are used in this spec.
-
-| Name | Value | Description |
-|---|---|---|
-| `GOSSIP_MAX_SIZE_BELLATRIX` | `10 * 2**20` (= 10,485,760, 10 MiB) | The maximum allowed size of uncompressed gossip messages starting at Bellatrix upgrade. |
-| `MAX_CHUNK_SIZE_BELLATRIX` | `10 * 2**20` (= 10,485,760, 10 MiB) | The maximum allowed size of uncompressed req/resp chunked responses starting at Bellatrix upgrade. |
-
 ### The gossip domain: gossipsub
 
 Some gossip meshes are upgraded in Bellatrix to support upgraded types.
@@ -61,11 +51,6 @@ All topics remain stable except the beacon block topic which is updated with the
 
 The specification around the creation, validation, and dissemination of messages has not changed from the Phase 0 and Altair documents unless explicitly noted here.
 
-Starting at Bellatrix upgrade, each gossipsub [message](https://github.com/libp2p/go-libp2p-pubsub/blob/master/pb/rpc.proto#L17-L24)
-has a maximum size of `GOSSIP_MAX_SIZE_BELLATRIX`.
-Clients MUST reject (fail validation) messages that are over this size limit.
-Likewise, clients MUST NOT emit or propagate messages larger than this limit.
-
 The derivation of the `message-id` remains stable.
 
 The new topics along with the type of the `data` field of a gossipsub message are given in this table:
@@ -130,10 +115,6 @@ down-scoring or disconnection.
 
 Request and Response remain unchanged unless explicitly noted here.
 
-Starting at Bellatrix upgrade,
-a global maximum uncompressed byte size of `MAX_CHUNK_SIZE_BELLATRIX` MUST be applied to all method response chunks
-regardless of type specific bounds that *MUST* also be respected.
-
 Bellatrix fork-digest is introduced to the `context` enum to specify Bellatrix block type.
 
 Per `context = compute_fork_digest(fork_version, genesis_validators_root)`:
@@ -171,17 +152,12 @@ Per `context = compute_fork_digest(fork_version, genesis_validators_root)`:
 
 With the addition of `ExecutionPayload` to `BeaconBlock`s, there is a dynamic
 field -- `transactions` -- which can validly exceed the `GOSSIP_MAX_SIZE` limit (1 MiB) put in
-place at Phase 0. At the `GAS_LIMIT` (~30M) currently seen on mainnet in 2021, a single transaction
+place at Phase 0, so GOSSIP_MAX_SIZE has increased to 10 Mib on the network. 
+At the `GAS_LIMIT` (~30M) currently seen on mainnet in 2021, a single transaction
 filled entirely with data at a cost of 16 gas per byte can create a valid
 `ExecutionPayload` of ~2 MiB. Thus we need a size limit to at least account for
 current mainnet conditions.
 
-Geth currently has a [max gossip message size](https://github.com/ethereum/go-ethereum/blob/3ce9f6d96f38712f5d6756e97b59ccc20cc403b3/eth/protocols/eth/protocol.go#L49) of 10 MiB.
-To support backward compatibility with this previously defined network limit,
-we adopt `GOSSIP_MAX_SIZE_BELLATRIX` of 10 MiB for maximum gossip sizes at the
-point of Bellatrix and beyond. Note, that clients SHOULD still reject objects
-that exceed their maximum theoretical bounds which in most cases is less than `GOSSIP_MAX_SIZE_BELLATRIX`.
-
 Note, that due to additional size induced by the `BeaconBlock` contents (e.g.
 proposer signature, operations lists, etc) this does reduce the
 theoretical max valid `ExecutionPayload` (and `transactions` list) size as