Merge branch 'unstable' into jeluard/fix-contributing

CONTRIBUTING.md

@@ -66,7 +66,7 @@ If you observe following error running any of the test files that means you are
 
 ```sh
 cd packages/beacon-node
-LODESTAR_PRESET=minimal yarn vitest --run --bail 1 --config vitest.config.spec.ts test/spec/presets/sanity.test.ts -t attester_slashing
+LODESTAR_PRESET=minimal yarn vitest --run --bail 1 --config vitest.spec.config.ts test/spec/presets/sanity.test.ts -t attester_slashing
 ```
 
 ## Docker

lerna.json

@@ -4,7 +4,7 @@
   ],
   "npmClient": "yarn",
   "useNx": true,
-  "version": "1.14.0",
+  "version": "1.15.0",
   "stream": true,
   "command": {
     "version": {

packages/api/package.json

@@ -11,7 +11,7 @@
   "bugs": {
     "url": "https://github.com/ChainSafe/lodestar/issues"
   },
-  "version": "1.14.0",
+  "version": "1.15.0",
   "type": "module",
   "exports": {
     ".": {
@@ -70,10 +70,10 @@
   "dependencies": {
     "@chainsafe/persistent-merkle-tree": "^0.6.1",
     "@chainsafe/ssz": "^0.14.0",
-    "@lodestar/config": "^1.14.0",
-    "@lodestar/params": "^1.14.0",
-    "@lodestar/types": "^1.14.0",
-    "@lodestar/utils": "^1.14.0",
+    "@lodestar/config": "^1.15.0",
+    "@lodestar/params": "^1.15.0",
+    "@lodestar/types": "^1.15.0",
+    "@lodestar/utils": "^1.15.0",
     "eventsource": "^2.0.2",
     "qs": "^6.11.1"
   },

packages/api/src/beacon/routes/beacon/block.ts

@@ -229,8 +229,12 @@ export type Api = {
    * Retrieves BlobSidecar included in requested block.
    * @param blockId Block identifier.
    * Can be one of: "head" (canonical head in node's view), "genesis", "finalized", \<slot\>, \<hex encoded blockRoot with 0x prefix\>.
+   * @param indices Array of indices for blob sidecars to request for in the specified block. Returns all blob sidecars in the block if not specified.
    */
-  getBlobSidecars(blockId: BlockId): Promise<
+  getBlobSidecars(
+    blockId: BlockId,
+    indices?: number[]
+  ): Promise<
     ApiClientResponse<{
       [HttpStatusCode.OK]: {executionOptimistic: ExecutionOptimistic; data: deneb.BlobSidecars};
     }>
@@ -270,7 +274,7 @@ export type ReqTypes = {
   publishBlockV2: {body: unknown; query: {broadcast_validation?: string}};
   publishBlindedBlock: {body: unknown};
   publishBlindedBlockV2: {body: unknown; query: {broadcast_validation?: string}};
-  getBlobSidecars: BlockIdOnlyReq;
+  getBlobSidecars: {params: {block_id: string}; query: {indices?: number[]}};
 };
 
 export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api, ReqTypes> {
@@ -356,7 +360,16 @@ export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api,
         query: {broadcast_validation: Schema.String},
       },
     },
-    getBlobSidecars: blockIdOnlyReq,
+    getBlobSidecars: {
+      writeReq: (block_id, indices) => ({
+        params: {block_id: String(block_id)},
+        query: {indices},
+      }),
+      parseReq: ({params, query}) => [params.block_id, query.indices],
+      schema: {
+        query: {indices: Schema.UintArray},
+      },
+    },
   };
 }
 

packages/api/test/unit/beacon/oapiSpec.test.ts

@@ -130,12 +130,6 @@ const ignoredProperties: Record<string, IgnoredProperty> = {
    */
   getHealth: {request: ["query.syncing_status"]},
 
-  /**
-   * https://github.com/ChainSafe/lodestar/issues/6185
-   *  - must have required property 'query'
-   */
-  getBlobSidecars: {request: ["query"]},
-
   /* 
    https://github.com/ChainSafe/lodestar/issues/4638 
    /query - must have required property 'skip_randao_verification'

packages/api/test/unit/beacon/testData/beacon.ts

@@ -71,7 +71,7 @@ export const testData: GenericServerTestCases<Api> = {
     res: undefined,
   },
   getBlobSidecars: {
-    args: ["head"],
+    args: ["head", [0]],
     res: {executionOptimistic: true, data: ssz.deneb.BlobSidecars.defaultValue()},
   },
 

packages/api/test/utils/checkAgainstSpec.ts

@@ -199,12 +199,21 @@ function prettyAjvErrors(errors: ErrorObject[] | null | undefined): string {
   return errors.map((e) => `${e.instancePath ?? "."} - ${e.message}`).join("\n");
 }
 
+type StringifiedProperty = string | StringifiedProperty[];
+
+function stringifyProperty(value: unknown): StringifiedProperty {
+  if (typeof value === "number") {
+    return value.toString(10);
+  } else if (Array.isArray(value)) {
+    return value.map(stringifyProperty);
+  }
+  return String(value);
+}
+
 function stringifyProperties(obj: Record<string, unknown>): Record<string, unknown> {
   for (const key of Object.keys(obj)) {
     const value = obj[key];
-    if (typeof value === "number") {
-      obj[key] = value.toString(10);
-    }
+    obj[key] = stringifyProperty(value);
   }
 
   return obj;

packages/beacon-node/package.json

@@ -11,7 +11,7 @@
   "bugs": {
     "url": "https://github.com/ChainSafe/lodestar/issues"
   },
-  "version": "1.14.0",
+  "version": "1.15.0",
   "type": "module",
   "exports": {
     ".": {
@@ -79,17 +79,17 @@
     "test:unit:minimal": "vitest --run --segfaultRetry 3 --dir test/unit/",
     "test:unit:mainnet": "LODESTAR_PRESET=mainnet vitest --run --dir test/unit-mainnet",
     "test:unit": "wrapper() { yarn test:unit:minimal $@ && yarn test:unit:mainnet $@; }; wrapper",
-    "test:e2e": "LODESTAR_PRESET=minimal vitest --run --segfaultRetry 3 --config vitest.config.e2e.ts --dir test/e2e",
+    "test:e2e": "LODESTAR_PRESET=minimal vitest --run --segfaultRetry 3 --config vitest.e2e.config.ts --dir test/e2e",
     "test:sim": "vitest --run test/sim/**/*.test.ts",
     "test:sim:merge-interop": "vitest --run test/sim/merge-interop.test.ts",
     "test:sim:mergemock": "vitest --run test/sim/mergemock.test.ts",
     "test:sim:withdrawals": "vitest --run test/sim/withdrawal-interop.test.ts",
     "test:sim:blobs": "vitest --run test/sim/4844-interop.test.ts",
     "download-spec-tests": "node --loader=ts-node/esm test/spec/downloadTests.ts",
-    "test:spec:bls": "vitest --run --config vitest.config.spec.ts --dir test/spec/bls/",
-    "test:spec:general": "vitest --run --config vitest.config.spec.ts --dir test/spec/general/",
-    "test:spec:minimal": "LODESTAR_PRESET=minimal vitest --run --config vitest.config.spec.ts --dir test/spec/presets/",
-    "test:spec:mainnet": "LODESTAR_PRESET=mainnet vitest --run --config vitest.config.spec.ts --dir test/spec/presets/",
+    "test:spec:bls": "vitest --run --config vitest.spec.config.ts --dir test/spec/bls/",
+    "test:spec:general": "vitest --run --config vitest.spec.config.ts --dir test/spec/general/",
+    "test:spec:minimal": "LODESTAR_PRESET=minimal vitest --run --config vitest.spec.config.ts --dir test/spec/presets/",
+    "test:spec:mainnet": "LODESTAR_PRESET=mainnet vitest --run --config vitest.spec.config.ts --dir test/spec/presets/",
     "test:spec": "yarn test:spec:bls && yarn test:spec:general && yarn test:spec:minimal && yarn test:spec:mainnet",
     "check-readme": "typescript-docs-verifier"
   },
@@ -120,18 +120,18 @@
     "@libp2p/peer-id-factory": "^4.0.3",
     "@libp2p/prometheus-metrics": "^3.0.10",
     "@libp2p/tcp": "9.0.10",
-    "@lodestar/api": "^1.14.0",
-    "@lodestar/config": "^1.14.0",
-    "@lodestar/db": "^1.14.0",
-    "@lodestar/fork-choice": "^1.14.0",
-    "@lodestar/light-client": "^1.14.0",
-    "@lodestar/logger": "^1.14.0",
-    "@lodestar/params": "^1.14.0",
-    "@lodestar/reqresp": "^1.14.0",
-    "@lodestar/state-transition": "^1.14.0",
-    "@lodestar/types": "^1.14.0",
-    "@lodestar/utils": "^1.14.0",
-    "@lodestar/validator": "^1.14.0",
+    "@lodestar/api": "^1.15.0",
+    "@lodestar/config": "^1.15.0",
+    "@lodestar/db": "^1.15.0",
+    "@lodestar/fork-choice": "^1.15.0",
+    "@lodestar/light-client": "^1.15.0",
+    "@lodestar/logger": "^1.15.0",
+    "@lodestar/params": "^1.15.0",
+    "@lodestar/reqresp": "^1.15.0",
+    "@lodestar/state-transition": "^1.15.0",
+    "@lodestar/types": "^1.15.0",
+    "@lodestar/utils": "^1.15.0",
+    "@lodestar/validator": "^1.15.0",
     "@multiformats/multiaddr": "^12.1.3",
     "@types/datastore-level": "^3.0.0",
     "buffer-xor": "^2.0.2",

packages/beacon-node/src/api/impl/beacon/blocks/index.ts

@@ -406,7 +406,7 @@ export function getBeaconBlockApi({
       await publishBlock(signedBlockOrContents, opts);
     },
 
-    async getBlobSidecars(blockId) {
+    async getBlobSidecars(blockId, indices) {
       const {block, executionOptimistic} = await resolveBlockId(chain, blockId);
       const blockRoot = config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message);
 
@@ -418,9 +418,10 @@ export function getBeaconBlockApi({
       if (!blobSidecars) {
         throw Error(`blobSidecars not found in db for slot=${block.message.slot} root=${toHexString(blockRoot)}`);
       }
+
       return {
         executionOptimistic,
-        data: blobSidecars,
+        data: indices ? blobSidecars.filter(({index}) => indices.includes(index)) : blobSidecars,
       };
     },
   };

packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts

@@ -319,7 +319,7 @@ export async function verifyBlockExecutionPayload(
       const lvhResponse = {
         executionStatus,
         latestValidExecHash: execResult.latestValidHash,
-        invalidateFromBlockHash: toHexString(block.message.parentRoot),
+        invalidateFromParentBlockRoot: toHexString(block.message.parentRoot),
       };
       const execError = new BlockError(block, {
         code: BlockErrorCode.EXECUTION_ENGINE_ERROR,
@@ -416,7 +416,7 @@ function getSegmentErrorResponse(
       invalidSegmentLVH = {
         executionStatus: ExecutionStatus.Invalid,
         latestValidExecHash: lvhResponse.latestValidExecHash,
-        invalidateFromBlockHash: parentBlock.blockRoot,
+        invalidateFromParentBlockRoot: parentBlock.blockRoot,
       };
     }
   }

packages/beacon-node/src/network/reqresp/score.ts

@@ -24,6 +24,7 @@ export function onOutgoingReqRespError(e: RequestError, method: ReqRespMethod):
   switch (e.type.code) {
     case RequestErrorCode.INVALID_REQUEST:
     case RequestErrorCode.INVALID_RESPONSE_SSZ:
+    case RequestErrorCode.SSZ_OVER_MAX_SIZE:
       return PeerAction.LowToleranceError;
 
     case RequestErrorCode.SERVER_ERROR:

packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts

@@ -12,7 +12,7 @@ import {waitForEvent} from "../../utils/events/resolver.js";
 import {getAndInitDevValidators} from "../../utils/node/validator.js";
 import {ChainEvent} from "../../../src/chain/index.js";
 import {NetworkEvent} from "../../../src/network/index.js";
-import {connect} from "../../utils/network.js";
+import {connect, onPeerConnect} from "../../utils/network.js";
 import {testLogger, LogLevel, TestLoggerOpts} from "../../utils/logger.js";
 import {BlockError, BlockErrorCode} from "../../../src/chain/errors/index.js";
 import {BlockSource, getBlockInput} from "../../../src/chain/blocks/types.js";
@@ -48,7 +48,7 @@ describe("sync / unknown block sync", function () {
   for (const {id, event} of testCases) {
     it(id, async function () {
       // the node needs time to transpile/initialize bls worker threads
-      const genesisSlotsDelay = 7;
+      const genesisSlotsDelay = 4;
       const genesisTime = Math.floor(Date.now() / 1000) + genesisSlotsDelay * testParams.SECONDS_PER_SLOT;
       const testLoggerOpts: TestLoggerOpts = {
         level: LogLevel.info,
@@ -71,6 +71,7 @@ describe("sync / unknown block sync", function () {
           chain: {blsVerifyAllMainThread: true},
         },
         validatorCount,
+        genesisTime,
         logger: loggerNodeA,
       });
 
@@ -100,7 +101,7 @@ describe("sync / unknown block sync", function () {
           chain: {blsVerifyAllMainThread: true},
         },
         validatorCount,
-        genesisTime: bn.chain.getHeadState().genesisTime,
+        genesisTime,
         logger: loggerNodeB,
       });
 
@@ -116,7 +117,11 @@ describe("sync / unknown block sync", function () {
         ({block}) => block === headSummary.blockRoot
       );
 
+      const connected = Promise.all([onPeerConnect(bn2.network), onPeerConnect(bn.network)]);
       await connect(bn2.network, bn.network);
+      await connected;
+      loggerNodeA.info("Node A connected to Node B");
+
       const headInput = getBlockInput.preDeneb(config, head, BlockSource.gossip, null);
 
       switch (event) {
@@ -147,4 +152,4 @@ describe("sync / unknown block sync", function () {
       await waitForSynced;
     });
   }
-}, {timeout: 30_000});
+}, {timeout: 40_000});

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@chainsafe/lodestar",
-  "version": "1.14.0",
+  "version": "1.15.0",
   "description": "Command line interface for lodestar",
   "author": "ChainSafe Systems",
   "license": "LGPL-3.0",
@@ -31,7 +31,7 @@
     "lint": "eslint --color --ext .ts src/ test/",
     "lint:fix": "yarn run lint --fix",
     "test:unit": "vitest --run --dir test/unit/",
-    "test:e2e": "vitest --run --config vitest.config.e2e.ts --dir test/e2e/",
+    "test:e2e": "vitest --run --config vitest.e2e.config.ts --dir test/e2e/",
     "test:sim:multifork": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/multi_fork.test.ts",
     "test:sim:mixedclient": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/mixed_client.test.ts",
     "test:sim:endpoints": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/endpoints.test.ts",
@@ -65,19 +65,18 @@
     "@libp2p/crypto": "^3.0.4",
     "@libp2p/peer-id": "^4.0.4",
     "@libp2p/peer-id-factory": "^4.0.3",
-    "@lodestar/api": "^1.14.0",
-    "@lodestar/beacon-node": "^1.14.0",
-    "@lodestar/config": "^1.14.0",
-    "@lodestar/db": "^1.14.0",
-    "@lodestar/light-client": "^1.14.0",
-    "@lodestar/logger": "^1.14.0",
-    "@lodestar/params": "^1.14.0",
-    "@lodestar/state-transition": "^1.14.0",
-    "@lodestar/types": "^1.14.0",
-    "@lodestar/utils": "^1.14.0",
-    "@lodestar/validator": "^1.14.0",
+    "@lodestar/api": "^1.15.0",
+    "@lodestar/beacon-node": "^1.15.0",
+    "@lodestar/config": "^1.15.0",
+    "@lodestar/db": "^1.15.0",
+    "@lodestar/light-client": "^1.15.0",
+    "@lodestar/logger": "^1.15.0",
+    "@lodestar/params": "^1.15.0",
+    "@lodestar/state-transition": "^1.15.0",
+    "@lodestar/types": "^1.15.0",
+    "@lodestar/utils": "^1.15.0",
+    "@lodestar/validator": "^1.15.0",
     "@multiformats/multiaddr": "^12.1.3",
-    "@types/lockfile": "^1.0.2",
     "bip39": "^3.1.0",
     "deepmerge": "^4.3.1",
     "ethers": "^6.7.0",
@@ -86,22 +85,23 @@
     "got": "^11.8.6",
     "inquirer": "^9.1.5",
     "js-yaml": "^4.1.0",
-    "lockfile": "^1.0.4",
     "lodash": "^4.17.21",
     "prom-client": "^15.1.0",
+    "proper-lockfile": "^4.1.2",
     "rimraf": "^4.4.1",
     "source-map-support": "^0.5.21",
     "uint8arrays": "^4.0.9",
     "uuidv4": "^6.2.13",
     "yargs": "^17.7.1"
   },
   "devDependencies": {
-    "@lodestar/test-utils": "^1.14.0",
+    "@lodestar/test-utils": "^1.15.0",
     "@types/debug": "^4.1.7",
     "@types/expand-tilde": "^2.0.0",
     "@types/got": "^9.6.12",
     "@types/inquirer": "^9.0.3",
     "@types/lodash": "^4.14.192",
+    "@types/proper-lockfile": "^4.1.4",
     "@types/yargs": "^17.0.24"
   }
 }