Merge branch 'unstable' into nh/prover-tests

ChainSafe · Nov 24, 2023 · 9d81528 · 9d81528
2 parents b3b6f72 + 228d7ba
commit 9d81528
Show file tree

Hide file tree

Showing 99 changed files with 2,286 additions and 392 deletions.
diff --git a/.github/actions/core-dump/action.yml b/.github/actions/core-dump/action.yml
@@ -0,0 +1,13 @@
+name: 'Take core dump files'
+description: 'List down and upload core dumps as artifacts'
+runs:
+  using: "composite"
+  steps:
+    - run: ls -l
+      shell: sh
+
+    - name: Backup core dump 
+      uses: actions/upload-artifact@v2
+      with:
+        name: core-dump
+        path: core.*
diff --git a/.github/actions/setup-debug-node/action.yml b/.github/actions/setup-debug-node/action.yml
@@ -0,0 +1,39 @@
+name: 'Setup node with debug support'
+description: 'Setup the nodejs version with debug support'
+inputs:
+  node-version:
+    description: 'Version of nodejs'
+    required: true
+    default: '20'
+  debug: 
+    description: Enable the debug version
+    required: true
+    default: 'false'
+
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-node@v3
+      with:
+        node-version: ${{ inputs.node-version }}
+        check-latest: true
+        cache: yarn 
+
+    # For now we only have the Node 20 debug build
+    - run: sudo apt-get install unzip && curl -L "https://drive.google.com/uc?export=download&id=1hlhbbQi-NJi8_WjULvOdo-K_tfZFzN3Z&confirm=t" > nodejs.zip && unzip nodejs.zip
+      shell: sh
+      if: ${{ inputs.debug == 'true' }}      
+    - run: sudo cp -f node $(which node)
+      shell: sh
+      if: ${{ inputs.debug == 'true' }}
+    - run: sudo sh -c "ulimit -c unlimited"
+      shell: sh
+      if: ${{ inputs.debug == 'true' }}
+    - run: sudo sh -c "echo core > /proc/sys/kernel/core_pattern"
+      shell: sh
+      if: ${{ inputs.debug == 'true' }}
+    - run: echo $(node --print "process.version")
+      shell: sh
+    - run: echo $(node --print "process.features.debug") 
+      shell: sh
+
diff --git a/.github/workflows/build-debug-node.yml b/.github/workflows/build-debug-node.yml
@@ -29,7 +29,7 @@ jobs:
         working-directory: 'nodejs'
 
       - name: Compile the nodejs
-        run: make -j4
+        run: make -j$(nproc --all)
         working-directory: 'nodejs'
 
       - name: Verify the build 
@@ -39,16 +39,12 @@ jobs:
       - name: Create destination folder 
         run: mkdir -p ${{ github.workspace }}/nodejs-debug-build-${{ github.event.inputs.version }}
 
-      - name: Copy nodejs build
-        run: make install 
+     - name: Copy nodejs debug build
+        run: cp out/Debug/node ${{ github.workspace }}/nodejs-debug-build-${{ github.event.inputs.version }}
         working-directory: 'nodejs'
-        env: 
-          DESTDIR: ${{ github.workspace }}/nodejs-debug-build-${{ github.event.inputs.version }}
 
       - name: Upload build to artifacts
         uses: actions/upload-artifact@v3
         with:
           name: nodejs-debug-build-${{ github.event.inputs.version }}
           path: nodejs-debug-build-${{ github.event.inputs.version }}
-
-
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -153,12 +153,12 @@ jobs:
       matrix:
         node: [20]
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-node@v3
-        with:
+      - uses: actions/checkout@v3      
+      - uses: './.github/actions/setup-debug-node'
+        with: 
           node-version: ${{matrix.node}}
-          check-latest: true
-          cache: yarn
+          debug: 'true'
+
       - name: Restore build cache
         id: cache-primes-restore
         uses: actions/cache/restore@v3
@@ -181,7 +181,12 @@ jobs:
       - name: Rebuild native modules with debug 
         run: npm rebuild --debug
       - name: Unit tests
+        id: unit_tests
         run: yarn test:unit
+
+      - uses: './.github/actions/core-dump'
+        if: ${{ failure() && steps.unit_tests.conclusion == 'failure' }}
+
       - name: Upload coverage data
         run: yarn coverage
 

diff --git a/dashboards/lodestar_networking.json b/dashboards/lodestar_networking.json
@@ -1,12 +1,12 @@
 {
   "__inputs": [
     {
-      "name": "DS_PROMETHEUS",
-      "type": "datasource",
-      "label": "Prometheus",
       "description": "",
+      "label": "Prometheus",
+      "name": "DS_PROMETHEUS",
       "pluginId": "prometheus",
-      "pluginName": "Prometheus"
+      "pluginName": "Prometheus",
+      "type": "datasource"
     }
   ],
   "annotations": {

diff --git a/dashboards/lodestar_validator_monitor.json b/dashboards/lodestar_validator_monitor.json
@@ -112,6 +112,7 @@
         "type": "prometheus",
         "uid": "${DS_PROMETHEUS}"
       },
+      "description": "https://hackmd.io/@dapplion/lodestar_attestation_summary",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -166,6 +167,7 @@
         "type": "prometheus",
         "uid": "${DS_PROMETHEUS}"
       },
+      "description": "https://hackmd.io/@dapplion/lodestar_attestation_summary",
       "fieldConfig": {
         "defaults": {
           "color": {

diff --git a/lerna.json b/lerna.json
@@ -1,8 +1,10 @@
 {
-  "packages": ["packages/*"],
+  "packages": [
+    "packages/*"
+  ],
   "npmClient": "yarn",
   "useNx": true,
-  "version": "1.11.3",
+  "version": "1.12.0",
   "stream": true,
   "command": {
     "version": {

diff --git a/packages/api/package.json b/packages/api/package.json
@@ -11,7 +11,7 @@
   "bugs": {
     "url": "https://github.com/ChainSafe/lodestar/issues"
   },
-  "version": "1.11.3",
+  "version": "1.12.0",
   "type": "module",
   "exports": {
     ".": {
@@ -70,11 +70,11 @@
   },
   "dependencies": {
     "@chainsafe/persistent-merkle-tree": "^0.6.1",
-    "@chainsafe/ssz": "^0.13.0",
-    "@lodestar/config": "^1.11.3",
-    "@lodestar/params": "^1.11.3",
-    "@lodestar/types": "^1.11.3",
-    "@lodestar/utils": "^1.11.3",
+    "@chainsafe/ssz": "^0.14.0",
+    "@lodestar/config": "^1.12.0",
+    "@lodestar/params": "^1.12.0",
+    "@lodestar/types": "^1.12.0",
+    "@lodestar/utils": "^1.12.0",
     "eventsource": "^2.0.2",
     "qs": "^6.11.1"
   },

diff --git a/packages/api/src/beacon/routes/beacon/block.ts b/packages/api/src/beacon/routes/beacon/block.ts
@@ -193,7 +193,7 @@ export type Api = {
 
   publishBlockV2(
     blockOrContents: allForks.SignedBeaconBlockOrContents,
-    opts: {broadcastValidation?: BroadcastValidation}
+    opts?: {broadcastValidation?: BroadcastValidation}
   ): Promise<
     ApiClientResponse<
       {
@@ -341,7 +341,7 @@ export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api,
     getBlockRoot: blockIdOnlyReq,
     publishBlock: reqOnlyBody(AllForksSignedBlockOrContents, Schema.Object),
     publishBlockV2: {
-      writeReq: (item, {broadcastValidation}) => ({
+      writeReq: (item, {broadcastValidation} = {}) => ({
         body: AllForksSignedBlockOrContents.toJson(item),
         query: {broadcast_validation: broadcastValidation},
       }),

diff --git a/packages/api/src/beacon/routes/beacon/state.ts b/packages/api/src/beacon/routes/beacon/state.ts
@@ -109,6 +109,23 @@ export type Api = {
     >
   >;
 
+  /**
+   * Fetch the RANDAO mix for the requested epoch from the state identified by 'stateId'.
+   *
+   * @param stateId State identifier.
+   * Can be one of: "head" (canonical head in node's view), "genesis", "finalized", "justified", \<slot\>, \<hex encoded stateRoot with 0x prefix\>.
+   * @param epoch Fetch randao mix for the given epoch. If an epoch is not specified then the RANDAO mix for the state's current epoch will be returned.
+   */
+  getStateRandao(
+    stateId: StateId,
+    epoch?: Epoch
+  ): Promise<
+    ApiClientResponse<
+      {[HttpStatusCode.OK]: {data: {randao: Root}; executionOptimistic: ExecutionOptimistic}},
+      HttpStatusCode.BAD_REQUEST | HttpStatusCode.NOT_FOUND
+    >
+  >;
+
   /**
    * Get state finality checkpoints
    * Returns finality checkpoints for state with given 'stateId'.
@@ -216,6 +233,7 @@ export const routesData: RoutesData<Api> = {
   getStateFinalityCheckpoints: {url: "/eth/v1/beacon/states/{state_id}/finality_checkpoints", method: "GET"},
   getStateFork: {url: "/eth/v1/beacon/states/{state_id}/fork", method: "GET"},
   getStateRoot: {url: "/eth/v1/beacon/states/{state_id}/root", method: "GET"},
+  getStateRandao: {url: "/eth/v1/beacon/states/{state_id}/randao", method: "GET"},
   getStateValidator: {url: "/eth/v1/beacon/states/{state_id}/validators/{validator_id}", method: "GET"},
   getStateValidators: {url: "/eth/v1/beacon/states/{state_id}/validators", method: "GET"},
   getStateValidatorBalances: {url: "/eth/v1/beacon/states/{state_id}/validator_balances", method: "GET"},
@@ -231,6 +249,7 @@ export type ReqTypes = {
   getStateFinalityCheckpoints: StateIdOnlyReq;
   getStateFork: StateIdOnlyReq;
   getStateRoot: StateIdOnlyReq;
+  getStateRandao: {params: {state_id: StateId}; query: {epoch?: number}};
   getStateValidator: {params: {state_id: StateId; validator_id: ValidatorId}};
   getStateValidators: {params: {state_id: StateId}; query: {id?: ValidatorId[]; status?: ValidatorStatus[]}};
   getStateValidatorBalances: {params: {state_id: StateId}; query: {id?: ValidatorId[]}};
@@ -266,6 +285,15 @@ export function getReqSerializers(): ReqSerializers<Api, ReqTypes> {
     getStateFork: stateIdOnlyReq,
     getStateRoot: stateIdOnlyReq,
 
+    getStateRandao: {
+      writeReq: (state_id, epoch) => ({params: {state_id}, query: {epoch}}),
+      parseReq: ({params, query}) => [params.state_id, query.epoch],
+      schema: {
+        params: {state_id: Schema.StringRequired},
+        query: {epoch: Schema.Uint},
+      },
+    },
+
     getStateValidator: {
       writeReq: (state_id, validator_id) => ({params: {state_id, validator_id}}),
       parseReq: ({params}) => [params.state_id, params.validator_id],
@@ -299,6 +327,10 @@ export function getReturnTypes(): ReturnTypes<Api> {
     root: ssz.Root,
   });
 
+  const RandaoContainer = new ContainerType({
+    randao: ssz.Root,
+  });
+
   const FinalityCheckpoints = new ContainerType(
     {
       previousJustified: ssz.phase0.Checkpoint,
@@ -346,6 +378,7 @@ export function getReturnTypes(): ReturnTypes<Api> {
   return {
     getStateRoot: ContainerDataExecutionOptimistic(RootContainer),
     getStateFork: ContainerDataExecutionOptimistic(ssz.phase0.Fork),
+    getStateRandao: ContainerDataExecutionOptimistic(RandaoContainer),
     getStateFinalityCheckpoints: ContainerDataExecutionOptimistic(FinalityCheckpoints),
     getStateValidators: ContainerDataExecutionOptimistic(ArrayOf(ValidatorResponse)),
     getStateValidator: ContainerDataExecutionOptimistic(ValidatorResponse),

diff --git a/packages/api/src/builder/routes.ts b/packages/api/src/builder/routes.ts
@@ -1,6 +1,6 @@
 import {fromHexString, toHexString} from "@chainsafe/ssz";
 import {ssz, allForks, bellatrix, Slot, Root, BLSPubkey} from "@lodestar/types";
-import {ForkName, isForkExecution} from "@lodestar/params";
+import {ForkName, isForkExecution, isForkBlobs} from "@lodestar/params";
 import {ChainForkConfig} from "@lodestar/config";
 
 import {
@@ -34,11 +34,14 @@ export type Api = {
       HttpStatusCode.NOT_FOUND | HttpStatusCode.BAD_REQUEST
     >
   >;
-  submitBlindedBlock(
-    signedBlock: allForks.SignedBlindedBeaconBlockOrContents
-  ): Promise<
+  submitBlindedBlock(signedBlock: allForks.SignedBlindedBeaconBlockOrContents): Promise<
     ApiClientResponse<
-      {[HttpStatusCode.OK]: {data: allForks.ExecutionPayload; version: ForkName}},
+      {
+        [HttpStatusCode.OK]: {
+          data: allForks.ExecutionPayload | allForks.ExecutionPayloadAndBlobsBundle;
+          version: ForkName;
+        };
+      },
       HttpStatusCode.SERVICE_UNAVAILABLE
     >
   >;
@@ -84,8 +87,13 @@ export function getReturnTypes(): ReturnTypes<Api> {
     getHeader: WithVersion((fork: ForkName) =>
       isForkExecution(fork) ? ssz.allForksExecution[fork].SignedBuilderBid : ssz.bellatrix.SignedBuilderBid
     ),
-    submitBlindedBlock: WithVersion((fork: ForkName) =>
-      isForkExecution(fork) ? ssz.allForksExecution[fork].ExecutionPayload : ssz.bellatrix.ExecutionPayload
+    submitBlindedBlock: WithVersion<allForks.ExecutionPayload | allForks.ExecutionPayloadAndBlobsBundle>(
+      (fork: ForkName) =>
+        isForkBlobs(fork)
+          ? ssz.allForksBlobs[fork].ExecutionPayloadAndBlobsBundle
+          : isForkExecution(fork)
+          ? ssz.allForksExecution[fork].ExecutionPayload
+          : ssz.bellatrix.ExecutionPayload
     ),
   };
 }