Merge branch 'dev' into refactor-setup-py

.circleci/config.yml

@@ -176,7 +176,7 @@ jobs:
  - checkout
  - run:
  name: Check table of contents
- command: sudo npm install -g doctoc@2 && make check_toc
+ command: sudo npm install -g doctoc@2.2.0 && make check_toc
  codespell:
  docker:
  - image: circleci/python:3.9

.github/workflows/run-tests.yml

@@ -6,10 +6,8 @@ defaults:
 
 env:
  TEST_PRESET_TYPE: "minimal"
- DEFAULT_BRANCH: "dev"
 
-# Run tests on workflow_Dispatch
-on: 
+on:
  push:
  branches:
  - dev 
@@ -22,10 +20,6 @@ on:
  description: Type of test to run, either mainnet or minimal
  type: string
  required: true
- commitRef:
- description: The branch, tag or SHA to checkout and build from
- default: dev
- required: true 
  schedule:
  - cron: '0 0 * * *'
 
@@ -47,19 +41,15 @@ jobs:
  steps:
  - name: Checkout this repo
  uses: actions/checkout@v3.2.0
- with:
- ref: ${{ github.event.inputs.commitRef || env.DEFAULT_BRANCH }}
  - name: Check table of contents
- run: sudo npm install -g doctoc@2 && make check_toc
+ run: sudo npm install -g doctoc@2.2.0 && make check_toc
 
  codespell:
  runs-on: self-hosted
  needs: preclear
  steps:
  - name: Checkout this repo
  uses: actions/checkout@v3.2.0
- with:
- ref: ${{ github.event.inputs.commitRef || env.DEFAULT_BRANCH }}
  - name: Check codespell
  run: pip install 'codespell<3.0.0,>=2.0.0' --user && make codespell
 
@@ -69,8 +59,6 @@ jobs:
  steps:
  - name: Checkout this repo
  uses: actions/checkout@v3.2.0
- with:
- ref: ${{ github.event.inputs.commitRef || env.DEFAULT_BRANCH }}
  - name: Install pyspec requirements
  run: make install_test
  - name: Run linter for pyspec
@@ -87,8 +75,6 @@ jobs:
  steps:
  - name: Checkout this repo
  uses: actions/checkout@v3.2.0
- with:
- ref: ${{ github.event.inputs.commitRef || env.DEFAULT_BRANCH }}
  - name: set TEST_PRESET_TYPE
  if: github.event.inputs.test_preset_type != '' 
  run: |

.gitignore

@@ -4,6 +4,7 @@ venv
 .venvs
 .venv
 /.pytest_cache
+*.swp
 
 build/
 output/

configs/mainnet.yaml

@@ -53,6 +53,9 @@ DENEB_FORK_EPOCH: 18446744073709551615
 # EIP6110
 EIP6110_FORK_VERSION: 0x05000000 # temporary stub
 EIP6110_FORK_EPOCH: 18446744073709551615
+# WHISK
+WHISK_FORK_VERSION: 0x06000000 # temporary stub
+WHISK_FORK_EPOCH: 18446744073709551615
 
 
 # Time parameters
@@ -132,3 +135,5 @@ MAX_REQUEST_BLOCKS_DENEB: 128
 MAX_REQUEST_BLOB_SIDECARS: 768
 # `2**12` (= 4096 epochs, ~18 days)
 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096
+# `6`
+BLOB_SIDECAR_SUBNET_COUNT: 6

configs/minimal.yaml

@@ -52,6 +52,9 @@ DENEB_FORK_EPOCH: 18446744073709551615
 # EIP6110
 EIP6110_FORK_VERSION: 0x05000001
 EIP6110_FORK_EPOCH: 18446744073709551615
+# WHISK
+WHISK_FORK_VERSION: 0x06000001
+WHISK_FORK_EPOCH: 18446744073709551615
 
 
 # Time parameters
@@ -133,3 +136,5 @@ MAX_REQUEST_BLOCKS_DENEB: 128
 MAX_REQUEST_BLOB_SIDECARS: 768
 # `2**12` (= 4096 epochs, ~18 days)
 MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096
+# `6`
+BLOB_SIDECAR_SUBNET_COUNT: 6

presets/mainnet/whisk.yaml

@@ -0,0 +1,20 @@
+# Mainnet preset - Whisk
+
+# Misc
+# ---------------------------------------------------------------
+# `uint64(4)`
+CURDLEPROOFS_N_BLINDERS: 4
+# `uint64(2**14)`
+WHISK_CANDIDATE_TRACKERS_COUNT: 16384
+# `uint64(2**13)` must be < WHISK_CANDIDATE_TRACKERS_COUNT
+WHISK_PROPOSER_TRACKERS_COUNT: 8192
+# `Epoch(2**8)`
+WHISK_EPOCHS_PER_SHUFFLING_PHASE: 256
+# `uint64(2**7 - CURDLEPROOFS_N_BLINDERS)`
+WHISK_VALIDATORS_PER_SHUFFLE: 124
+# `Epoch(2)`
+WHISK_PROPOSER_SELECTION_GAP: 2
+# `uint64(2**15)` TODO: will be replaced by a fix format once there's a serialized format
+WHISK_MAX_SHUFFLE_PROOF_SIZE: 32768
+# `uint64(2**10)` TODO: will be replaced by a fix format once there's a serialized format
+WHISK_MAX_OPENING_PROOF_SIZE: 1024

presets/minimal/whisk.yaml

@@ -0,0 +1,20 @@
+# Minimal preset - Whisk
+
+# Misc
+# ---------------------------------------------------------------
+# [customized]
+CURDLEPROOFS_N_BLINDERS: 4
+# [customized]
+WHISK_CANDIDATE_TRACKERS_COUNT: 32
+# [customized]
+WHISK_PROPOSER_TRACKERS_COUNT: 16
+# [customized]
+WHISK_EPOCHS_PER_SHUFFLING_PHASE: 4
+# [customized]
+WHISK_VALIDATORS_PER_SHUFFLE: 4
+# [customized]
+WHISK_PROPOSER_SELECTION_GAP: 1
+# `uint64(2**15)` TODO: will be replaced by a fix format once there's a serialized format
+WHISK_MAX_SHUFFLE_PROOF_SIZE: 32768
+# `uint64(2**10)` TODO: will be replaced by a fix format once there's a serialized format
+WHISK_MAX_OPENING_PROOF_SIZE: 1024

pysetup/helpers.py

@@ -1,17 +1,27 @@
-from pathlib import Path
 import re
-from typing import TypeVar, Dict, Sequence
+from typing import TypeVar, Dict
 import textwrap
+from functools import reduce
 
 from .constants import CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS
-from .spec_builders import SpecBuilder
+from .spec_builders import spec_builders
+from .md_doc_paths import PREVIOUS_FORK_OF
 from .typing import (
  ProtocolDefinition,
  SpecObject,
  VariableDefinition,
 )
 
 
+def collect_prev_forks(fork: str) -> list[str]:
+ forks = [fork]
+ while True:
+ fork = PREVIOUS_FORK_OF[fork]
+ if fork is None:
+ return forks
+ forks.append(fork)
+
+
 def is_byte_vector(value: str) -> bool:
  return value.startswith(('ByteVector'))
 
@@ -23,7 +33,7 @@ def make_function_abstract(protocol_def: ProtocolDefinition, key: str):
 
 def objects_to_spec(preset_name: str,
  spec_object: SpecObject,
- builder: SpecBuilder,
+ fork: str,
  ordered_class_objects: Dict[str, str]) -> str:
  """
  Given all the objects that constitute a spec, combine them into a single pyfile.
@@ -37,6 +47,10 @@ def objects_to_spec(preset_name: str,
  )
  )
 
+ # Collect builders with the reversed previous forks
+ # e.g. `[bellatrix, altair, phase0]` -> `[phase0, altair, bellatrix]`
+ builders = [spec_builders[fork] for fork in collect_prev_forks(fork)[::-1]]
+
  def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str:
  abstract_functions = ["verify_and_notify_new_payload"]
  for key in protocol_def.functions.keys():
@@ -58,7 +72,8 @@ def format_protocol(protocol_name: str, protocol_def: ProtocolDefinition) -> str
  "compute_merkle_proof_for_state",
  ]:
  del spec_object.functions[k]
- functions = builder.implement_optimizations(spec_object.functions)
+
+ functions = reduce(lambda fns, builder: builder.implement_optimizations(fns), builders, spec_object.functions)
  functions_spec = '\n\n\n'.join(functions.values())
 
  # Access global dict of config vars for runtime configurables
@@ -94,37 +109,46 @@ def format_constant(name: str, vardef: VariableDefinition) -> str:
  if vardef.comment is not None:
  out += f' # {vardef.comment}'
  return out
+
+ # Merge all constant objects
+ hardcoded_ssz_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_ssz_dep_constants()}, builders, {})
+ hardcoded_custom_type_dep_constants = reduce(lambda obj, builder: {**obj, **builder.hardcoded_custom_type_dep_constants(spec_object)}, builders, {})
+ # Concatenate all strings
+ imports = reduce(lambda txt, builder: (txt + "\n\n" + builder.imports(preset_name) ).strip("\n"), builders, "")
+ preparations = reduce(lambda txt, builder: (txt + "\n\n" + builder.preparations() ).strip("\n"), builders, "")
+ sundry_functions = reduce(lambda txt, builder: (txt + "\n\n" + builder.sundry_functions() ).strip("\n"), builders, "")
+ # Keep engine from the most recent fork
+ execution_engine_cls = reduce(lambda txt, builder: builder.execution_engine_cls() or txt, builders, "")
 
  constant_vars_spec = '# Constant vars\n' + '\n'.join(format_constant(k, v) for k, v in spec_object.constant_vars.items())
  preset_vars_spec = '# Preset vars\n' + '\n'.join(format_constant(k, v) for k, v in spec_object.preset_vars.items())
  ordered_class_objects_spec = '\n\n\n'.join(ordered_class_objects.values())
- ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_ssz_dep_constants()[x]), builder.hardcoded_ssz_dep_constants()))
- ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), builder.hardcoded_ssz_dep_constants()))
- custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, builder.hardcoded_custom_type_dep_constants(spec_object)[x]), builder.hardcoded_custom_type_dep_constants(spec_object)))
- spec = (
- builder.imports(preset_name)
- + builder.preparations()
- + '\n\n' + f"fork = \'{builder.fork}\'\n"
+ ssz_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
+ ssz_dep_constants_verification = '\n'.join(map(lambda x: 'assert %s == %s' % (x, spec_object.ssz_dep_constants[x]), hardcoded_ssz_dep_constants))
+ custom_type_dep_constants = '\n'.join(map(lambda x: '%s = %s' % (x, hardcoded_custom_type_dep_constants[x]), hardcoded_custom_type_dep_constants))
+ spec_strs = [
+ imports,
+ preparations,
+ f"fork = \'{fork}\'\n",
  # The constants that some SSZ containers require. Need to be defined before `new_type_definitions`
- + ('\n\n' + custom_type_dep_constants + '\n' if custom_type_dep_constants != '' else '')
- + '\n\n' + new_type_definitions
- + '\n' + CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS
+ custom_type_dep_constants,
+ new_type_definitions,
+ CONSTANT_DEP_SUNDRY_CONSTANTS_FUNCTIONS,
  # The constants that some SSZ containers require. Need to be defined before `constants_spec`
- + ('\n\n' + ssz_dep_constants if ssz_dep_constants != '' else '')
- + '\n\n' + constant_vars_spec
- + '\n\n' + preset_vars_spec
- + '\n\n\n' + config_spec
- + '\n\n' + ordered_class_objects_spec
- + ('\n\n\n' + protocols_spec if protocols_spec != '' else '')
- + '\n\n\n' + functions_spec
- + '\n\n' + builder.sundry_functions()
- + builder.execution_engine_cls()
+ ssz_dep_constants,
+ constant_vars_spec,
+ preset_vars_spec,
+ config_spec,
+ ordered_class_objects_spec,
+ protocols_spec,
+ functions_spec,
+ sundry_functions,
+ execution_engine_cls,
  # Since some constants are hardcoded in setup.py, the following assertions verify that the hardcoded constants are
  # as same as the spec definition.
- + ('\n\n\n' + ssz_dep_constants_verification if ssz_dep_constants_verification != '' else '')
- + '\n'
- )
- return spec
+ ssz_dep_constants_verification,
+ ]
+ return "\n\n\n".join([str.strip("\n") for str in spec_strs if str]) + "\n"
 
 
 def combine_protocols(old_protocols: Dict[str, ProtocolDefinition],

pysetup/md_doc_paths.py

@@ -47,6 +47,7 @@ def is_post_fork(a, b) -> bool:
  else:
  return is_post_fork(prev_fork, b)
 
+
 def get_fork_directory(fork):
  dir1 = f'specs/{fork}'
  if os.path.exists(dir1):