Merge branch 'unstable' into cayman/historical-state-regen

.dockerignore

@@ -46,11 +46,11 @@ packages/beacon-node/mainnet_pubkeys.csv
 # Autogenerated docs
 packages/**/docs
 packages/**/typedocs
-docs/packages
-docs/contributing.md
-docs/assets
-docs/reference/cli.md
-/site
+docs/pages/**/*-cli.md
+docs/pages/assets
+docs/pages/api/api-reference.md
+docs/pages/contribution/getting-started.md
+docs/site
 
 # Lodestar artifacts
 .lodestar

.github/actions/core-dump/action.yml

@@ -0,0 +1,16 @@
+name: 'Take core dump files'
+description: 'List down and upload core dumps as artifacts'
+runs:
+  using: "composite"
+  steps:
+    - name: List down core dump files 
+      run: |
+        ls -lah /cores/
+        sudo chmod -R +rwx /cores/*
+      shell: sh
+
+    - name: Backup core dump 
+      uses: actions/upload-artifact@v3
+      with:
+        name: core-dump
+        path: /cores/*

.github/actions/setup-debug-node/action.yml

@@ -0,0 +1,23 @@
+name: "Setup node with debug support"
+description: "Setup the nodejs version with debug support"
+runs:
+  using: "composite"
+  steps:
+    # For now we only have the Node 20 debug build
+    - run: |
+        sudo apt-get install unzip && curl -L "https://drive.google.com/uc?export=download&id=1hlhbbQi-NJi8_WjULvOdo-K_tfZFzN3Z&confirm=t" > nodejs.zip && unzip nodejs.zip
+        sudo cp -f node /usr/bin/node-with-debug
+        sudo chmod +x /usr/bin/node-with-debug
+      shell: sh
+
+    # List of naming patterns
+    # https://man7.org/linux/man-pages/man5/core.5.html
+    - run: |
+        sudo mkdir -p /cores
+        sudo sh -c "echo /cores/core-%e-%s-%u-%g-%p-%t > /proc/sys/kernel/core_pattern"
+      shell: sh
+
+    - run: |
+        echo $(/usr/bin/node-with-debug --print "process.version")
+        echo $(/usr/bin/node-with-debug --print "process.features.debug")
+      shell: sh

.github/workflows/benchmark.yml

@@ -17,6 +17,7 @@ on:
     branches:
       - stable
       - unstable
+  workflow_dispatch:
 
 jobs:
   run:

.github/workflows/build-debug-node.yml

@@ -0,0 +1,50 @@
+name: Build debug node 
+
+on: 
+  workflow_dispatch:
+    inputs:
+      version:
+        required: true
+        description: 'Node.js version'
+
+jobs:
+  build:
+    name: Build Debug version of Node.js
+    runs-on: buildjet-4vcpu-ubuntu-2204
+    strategy:
+      fail-fast: false
+    steps:
+      - name: Install dependencies 
+        run: apt-get install python3 g++ make python3-pip
+
+      - name: Download Node.js source
+        uses: actions/checkout@v4
+        with: 
+          repository: 'nodejs/node'
+          ref: 'v${{ github.event.inputs.version }}'
+          path: 'nodejs'
+
+      - name: Configure nodejs with debug flag 
+        run: ./configure --debug
+        working-directory: 'nodejs'
+
+      - name: Compile the nodejs
+        run: make -j$(nproc --all)
+        working-directory: 'nodejs'
+
+      - name: Verify the build 
+        run: make test-only 
+        working-directory: 'nodejs'
+
+      - name: Create destination folder 
+        run: mkdir -p ${{ github.workspace }}/nodejs-debug-build-${{ github.event.inputs.version }}
+
+      - name: Copy nodejs debug build
+        run: cp out/Debug/node ${{ github.workspace }}/nodejs-debug-build-${{ github.event.inputs.version }}
+        working-directory: 'nodejs'
+
+      - name: Upload build to artifacts
+        uses: actions/upload-artifact@v3
+        with:
+          name: nodejs-debug-build-${{ github.event.inputs.version }}
+          path: nodejs-debug-build-${{ github.event.inputs.version }}

.github/workflows/docs-check.yml

@@ -39,12 +39,12 @@ jobs:
         run: scripts/wordlist_sort_check.sh
 
       - name: Build and collect docs
-        run: yarn build:docs
+        run: yarn docs:build
 
       # Run prettier check with fix after generating the docs. The CLI reference is formatted with prettier for
       # deployed version so this will fail if not "fixable"
       - name: Check docs format
-        run: yarn lint-docs:fix
+        run: yarn docs:lint:fix
 
       # Run spellcheck AFTER building docs, in case the CLI reference has issues
       - name: Spellcheck

.github/workflows/docs.yml

@@ -4,21 +4,41 @@ on:
   push:
     branches:
       - stable
+  workflow_dispatch:
+    inputs:
+      ref:
+        description: 'Ref to deploy, defaults to `unstable`'
+        required: false
+        default: 'unstable'
+        type: string
 
 jobs:
   docs:
     runs-on: buildjet-4vcpu-ubuntu-2204
+    env:
+      DEPLOY_REF: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.ref || 'stable' }}
     steps:
-      # <common-build> - Uses YAML anchors in the future
+      # Log out the ref being deployed
+      - name: Log Deployment Ref
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          echo "Deploying ref: $DEPLOY_REF"
+
+      # Checkout the correct ref being deployed
       - uses: actions/checkout@v3
+        with:
+          ref: ${{ env.DEPLOY_REF }}
+
       - uses: actions/setup-node@v3
         with:
           node-version: 20
           check-latest: true
           cache: yarn          
+
       - name: Node.js version
         id: node
         run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT
+
       - name: Restore dependencies
         uses: actions/cache@master
         id: cache-deps
@@ -27,33 +47,36 @@ jobs:
             node_modules
             packages/*/node_modules
           key: ${{ runner.os }}-${{ steps.node.outputs.v8CppApiVersion }}-${{ hashFiles('**/yarn.lock', '**/package.json') }}
+
       - name: Install & build
         if: steps.cache-deps.outputs.cache-hit != 'true'
         run: yarn install --frozen-lockfile && yarn build
+
       - name: Build
         run: yarn build
         if: steps.cache-deps.outputs.cache-hit == 'true'
-      # </common-build>
 
       - name: Build and collect docs
-        run: yarn build:docs
+        run: yarn docs:build
 
       - name: Lint built docs
-        run: yarn lint-docs:fix
+        run: yarn docs:lint:fix
 
       - name: Set up Python
         uses: actions/setup-python@v1
 
       - name: Install dependencies
+        working-directory: docs
         run: |
           python -m pip install --upgrade pip
-          pip install -r docs/requirements.txt
+          pip install -r requirements.txt
 
       - name: Build docs
-        run: mkdocs build --site-dir site -v --clean
+        working-directory: docs
+        run: mkdocs build --verbose --clean --site-dir site
 
       - name: Deploy
         uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./site
+          publish_dir: ./docs/site

.github/workflows/test-sim.yml

@@ -16,6 +16,11 @@ on:
         description: Runtime DEBUG value
         required: false
         default: ""
+      genesisDelaySlots:
+        description: 'Number of slots delay before genesis'
+        required: true
+        type: number
+        default: 40
 
 env:
   GETH_DOCKER_IMAGE: ethereum/client-go:v1.11.6
@@ -56,6 +61,8 @@ jobs:
       - name: Sim tests multifork
         run: DEBUG='${{github.event.inputs.debug}}' yarn test:sim:multifork
         working-directory: packages/cli
+        env: 
+          GENESIS_DELAY_SLOTS: ${{github.event.inputs.genesisDelaySlots}}
 
       - name: Sim tests endpoints
         run: yarn test:sim:endpoints

.github/workflows/test.yml

@@ -153,12 +153,16 @@ jobs:
       matrix:
         node: [20]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v3      
       - uses: actions/setup-node@v3
         with:
-          node-version: ${{matrix.node}}
+          node-version: ${{ matrix.node }}
           check-latest: true
-          cache: yarn
+          cache: yarn 
+
+      # # Remove when finished debugging core dumps
+      # - uses: './.github/actions/setup-debug-node'
+
       - name: Restore build cache
         id: cache-primes-restore
         uses: actions/cache/restore@v3
@@ -180,7 +184,15 @@ jobs:
           key: spec-test-data-${{ hashFiles('packages/validator/test/spec/params.ts') }}
 
       - name: Unit tests
+        # id: unit_tests
+        # Rever to "yarn test:unit" when finished debugging core dumps
+        # run: sudo sh -c "ulimit -c unlimited && /usr/bin/node-with-debug $(which yarn) test:unit"
         run: yarn test:unit
+
+      # # Remove when finished debugging core dumps
+      # - uses: './.github/actions/core-dump'
+      #   if: ${{ failure() && steps.unit_tests.conclusion == 'failure' }}
+
       - name: Upload coverage data
         run: yarn coverage
 
@@ -259,7 +271,10 @@ jobs:
             packages/*/.git-data.json
           key: ${{ runner.os }}-node-${{ matrix.node }}-${{ github.sha }}
           fail-on-cache-miss: true
-
+      - name: Install Chrome browser
+        run: npx @puppeteer/browsers install chromedriver@latest --path /tmp          
+      - name: Install Firefox browser
+        run: npx @puppeteer/browsers install firefox@latest --path /tmp
       - name: Browser tests
         run: |
           export DISPLAY=':99.0'
@@ -306,12 +321,15 @@ jobs:
       # Run them in different steps to quickly identifying which command failed
       # Otherwise just doing `yarn test:spec` you can't tell which specific suite failed
       # many of the suites have identical names for minimal and mainnet
-      - name: Spec tests bls-general
-        run: yarn test:spec-bls-general
+      - name: Spec tests general
+        run: yarn test:spec:general
         working-directory: packages/beacon-node
+      - name: Spec tests bls
+        run: yarn test:spec:bls
+        working-directory: packages/beacon-node        
       - name: Spec tests minimal
-        run: yarn test:spec-minimal
+        run: yarn test:spec:minimal
         working-directory: packages/beacon-node
       - name: Spec tests mainnet
-        run: NODE_OPTIONS='--max-old-space-size=4096' yarn test:spec-mainnet
+        run: NODE_OPTIONS='--max-old-space-size=4096' yarn test:spec:mainnet
         working-directory: packages/beacon-node

.gitignore

@@ -40,11 +40,15 @@ packages/api/oapi-schemas
 # Autogenerated docs
 packages/**/docs
 packages/**/typedocs
-docs/assets
-docs/packages
-docs/reference
-docs/contributing.md
-/site
+docs/pages/**/*-cli.md
+docs/pages/assets
+docs/pages/images
+docs/pages/security.md
+docs/pages/lightclient-prover/lightclient.md
+docs/pages/lightclient-prover/prover.md
+docs/pages/api/api-reference.md
+docs/pages/contribution/getting-started.md
+docs/site
 
 # Testnet artifacts
 .lodestar