From 01fde1283fcc6b2a0a129e848528ec9d7443579d Mon Sep 17 00:00:00 2001 From: Phil Ngo Date: Fri, 19 Jul 2024 07:55:30 -0400 Subject: [PATCH 1/4] chore: bump package versions to 1.20.2 --- lerna.json | 2 +- packages/api/package.json | 10 +++++----- packages/beacon-node/package.json | 26 +++++++++++++------------- packages/cli/package.json | 26 +++++++++++++------------- packages/config/package.json | 6 +++--- packages/db/package.json | 8 ++++---- packages/flare/package.json | 14 +++++++------- packages/fork-choice/package.json | 12 ++++++------ packages/light-client/package.json | 12 ++++++------ packages/logger/package.json | 6 +++--- packages/params/package.json | 2 +- packages/prover/package.json | 18 +++++++++--------- packages/reqresp/package.json | 12 ++++++------ packages/spec-test-util/package.json | 4 ++-- packages/state-transition/package.json | 10 +++++----- packages/test-utils/package.json | 6 +++--- packages/types/package.json | 4 ++-- packages/utils/package.json | 2 +- packages/validator/package.json | 18 +++++++++--------- 19 files changed, 99 insertions(+), 99 deletions(-) diff --git a/lerna.json b/lerna.json index bfba68d36c54..ccdcaca872ec 100644 --- a/lerna.json +++ b/lerna.json @@ -4,7 +4,7 @@ ], "npmClient": "yarn", "useNx": true, - "version": "1.20.1", + "version": "1.20.2", "stream": true, "command": { "version": { diff --git a/packages/api/package.json b/packages/api/package.json index 280d4bf4e9ef..5b91b46c128e 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": { ".": { @@ -72,10 +72,10 @@ "dependencies": { "@chainsafe/persistent-merkle-tree": "^0.7.1", "@chainsafe/ssz": "^0.15.1", - "@lodestar/config": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/config": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2", "eventsource": "^2.0.2", "qs": "^6.11.1" }, diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index c57de6eb5781..80d61cb119cb 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": { ".": { @@ -120,18 +120,18 @@ "@libp2p/peer-id-factory": "^4.1.0", "@libp2p/prometheus-metrics": "^3.0.21", "@libp2p/tcp": "9.0.23", - "@lodestar/api": "^1.20.1", - "@lodestar/config": "^1.20.1", - "@lodestar/db": "^1.20.1", - "@lodestar/fork-choice": "^1.20.1", - "@lodestar/light-client": "^1.20.1", - "@lodestar/logger": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/reqresp": "^1.20.1", - "@lodestar/state-transition": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1", - "@lodestar/validator": "^1.20.1", + "@lodestar/api": "^1.20.2", + "@lodestar/config": "^1.20.2", + "@lodestar/db": "^1.20.2", + "@lodestar/fork-choice": "^1.20.2", + "@lodestar/light-client": "^1.20.2", + "@lodestar/logger": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/reqresp": "^1.20.2", + "@lodestar/state-transition": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2", + "@lodestar/validator": "^1.20.2", "@multiformats/multiaddr": "^12.1.3", "c-kzg": "^2.1.2", "datastore-core": "^9.1.1", diff --git a/packages/cli/package.json b/packages/cli/package.json index 4f27d030c735..30b446c9fd1b 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@chainsafe/lodestar", - "version": "1.20.1", + "version": "1.20.2", "description": "Command line interface for lodestar", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -63,17 +63,17 @@ "@libp2p/crypto": "^4.1.0", "@libp2p/peer-id": "^4.1.0", "@libp2p/peer-id-factory": "^4.1.0", - "@lodestar/api": "^1.20.1", - "@lodestar/beacon-node": "^1.20.1", - "@lodestar/config": "^1.20.1", - "@lodestar/db": "^1.20.1", - "@lodestar/light-client": "^1.20.1", - "@lodestar/logger": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/state-transition": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1", - "@lodestar/validator": "^1.20.1", + "@lodestar/api": "^1.20.2", + "@lodestar/beacon-node": "^1.20.2", + "@lodestar/config": "^1.20.2", + "@lodestar/db": "^1.20.2", + "@lodestar/light-client": "^1.20.2", + "@lodestar/logger": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/state-transition": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2", + "@lodestar/validator": "^1.20.2", "@multiformats/multiaddr": "^12.1.3", "deepmerge": "^4.3.1", "ethers": "^6.7.0", @@ -89,7 +89,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.20.1", + "@lodestar/test-utils": "^1.20.2", "@types/debug": "^4.1.7", "@types/got": "^9.6.12", "@types/inquirer": "^9.0.3", diff --git a/packages/config/package.json b/packages/config/package.json index f25d67bb687d..8d5fd3d80c35 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/config", - "version": "1.20.1", + "version": "1.20.2", "description": "Chain configuration required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -65,7 +65,7 @@ ], "dependencies": { "@chainsafe/ssz": "^0.15.1", - "@lodestar/params": "^1.20.1", - "@lodestar/types": "^1.20.1" + "@lodestar/params": "^1.20.2", + "@lodestar/types": "^1.20.2" } } diff --git a/packages/db/package.json b/packages/db/package.json index f883ff39291d..6cfaf6ecce70 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/db", - "version": "1.20.1", + "version": "1.20.2", "description": "DB modules of Lodestar", "author": "ChainSafe Systems", "homepage": "https://github.com/ChainSafe/lodestar#readme", @@ -36,12 +36,12 @@ }, "dependencies": { "@chainsafe/ssz": "^0.15.1", - "@lodestar/config": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/config": "^1.20.2", + "@lodestar/utils": "^1.20.2", "classic-level": "^1.4.1", "it-all": "^3.0.4" }, "devDependencies": { - "@lodestar/logger": "^1.20.1" + "@lodestar/logger": "^1.20.2" } } diff --git a/packages/flare/package.json b/packages/flare/package.json index b9a02da63383..c735642ad8e4 100644 --- a/packages/flare/package.json +++ b/packages/flare/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/flare", - "version": "1.20.1", + "version": "1.20.2", "description": "Beacon chain debugging tool", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -60,12 +60,12 @@ "dependencies": { "@chainsafe/bls": "7.1.3", "@chainsafe/bls-keygen": "^0.4.0", - "@lodestar/api": "^1.20.1", - "@lodestar/config": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/state-transition": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/api": "^1.20.2", + "@lodestar/config": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/state-transition": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2", "source-map-support": "^0.5.21", "yargs": "^17.7.1" }, diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index 8352c751aa37..2da624af4581 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": "./lib/index.js", "types": "./lib/index.d.ts", @@ -37,11 +37,11 @@ }, "dependencies": { "@chainsafe/ssz": "^0.15.1", - "@lodestar/config": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/state-transition": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1" + "@lodestar/config": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/state-transition": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2" }, "keywords": [ "ethereum", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index 68e8bf6a26dd..8b433375f07f 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": { ".": { @@ -76,11 +76,11 @@ "@chainsafe/bls": "7.1.3", "@chainsafe/persistent-merkle-tree": "^0.7.1", "@chainsafe/ssz": "^0.15.1", - "@lodestar/api": "^1.20.1", - "@lodestar/config": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/api": "^1.20.2", + "@lodestar/config": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2", "mitt": "^3.0.0" }, "devDependencies": { diff --git a/packages/logger/package.json b/packages/logger/package.json index 15e33b579ba0..117040580d60 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": { ".": { @@ -66,14 +66,14 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@lodestar/utils": "^1.20.1", + "@lodestar/utils": "^1.20.2", "winston": "^3.8.2", "winston-daily-rotate-file": "^4.7.1", "winston-transport": "^4.5.0" }, "devDependencies": { "@chainsafe/threads": "^1.11.1", - "@lodestar/test-utils": "^1.20.1", + "@lodestar/test-utils": "^1.20.2", "@types/triple-beam": "^1.3.2", "triple-beam": "^1.3.0" }, diff --git a/packages/params/package.json b/packages/params/package.json index c8505204c42e..20f50f625cfd 100644 --- a/packages/params/package.json +++ b/packages/params/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/params", - "version": "1.20.1", + "version": "1.20.2", "description": "Chain parameters required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", diff --git a/packages/prover/package.json b/packages/prover/package.json index 577e3bc105e2..f5d8c26105d4 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": { ".": { @@ -69,13 +69,13 @@ "@ethereumjs/tx": "^4.1.2", "@ethereumjs/util": "^8.0.6", "@ethereumjs/vm": "^6.4.2", - "@lodestar/api": "^1.20.1", - "@lodestar/config": "^1.20.1", - "@lodestar/light-client": "^1.20.1", - "@lodestar/logger": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/api": "^1.20.2", + "@lodestar/config": "^1.20.2", + "@lodestar/light-client": "^1.20.2", + "@lodestar/logger": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2", "ethereum-cryptography": "^2.0.0", "find-up": "^6.3.0", "http-proxy": "^1.18.1", @@ -84,7 +84,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.20.1", + "@lodestar/test-utils": "^1.20.2", "@types/http-proxy": "^1.17.10", "@types/yargs": "^17.0.24", "axios": "^1.3.4", diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index 8091e4f16ee7..37c16318c5b3 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": { ".": { @@ -54,9 +54,9 @@ "dependencies": { "@chainsafe/fast-crc32c": "^4.1.1", "@libp2p/interface": "^1.3.0", - "@lodestar/config": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/config": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/utils": "^1.20.2", "it-all": "^3.0.4", "it-pipe": "^3.0.1", "snappy": "^7.2.2", @@ -65,8 +65,8 @@ "uint8arraylist": "^2.4.7" }, "devDependencies": { - "@lodestar/logger": "^1.20.1", - "@lodestar/types": "^1.20.1", + "@lodestar/logger": "^1.20.2", + "@lodestar/types": "^1.20.2", "libp2p": "1.4.3" }, "peerDependencies": { diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json index f5f590d55fa0..bb2fe357c5f8 100644 --- a/packages/spec-test-util/package.json +++ b/packages/spec-test-util/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/spec-test-util", - "version": "1.20.1", + "version": "1.20.2", "description": "Spec test suite generator from yaml test files", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -62,7 +62,7 @@ "blockchain" ], "dependencies": { - "@lodestar/utils": "^1.20.1", + "@lodestar/utils": "^1.20.2", "axios": "^1.3.4", "rimraf": "^4.4.1", "snappyjs": "^0.7.0", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index 7dc542762cbf..b032f5204af3 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": { ".": { @@ -64,10 +64,10 @@ "@chainsafe/persistent-merkle-tree": "^0.7.1", "@chainsafe/persistent-ts": "^0.19.1", "@chainsafe/ssz": "^0.15.1", - "@lodestar/config": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/config": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2", "bigint-buffer": "^1.1.5" }, "keywords": [ diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index 45fb3d0e0909..d2571a974b65 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -1,7 +1,7 @@ { "name": "@lodestar/test-utils", "private": true, - "version": "1.20.1", + "version": "1.20.2", "description": "Test utilities reused across other packages", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -59,8 +59,8 @@ "dependencies": { "@chainsafe/bls": "7.1.3", "@chainsafe/bls-keystore": "^3.0.1", - "@lodestar/params": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/params": "^1.20.2", + "@lodestar/utils": "^1.20.2", "axios": "^1.3.4", "testcontainers": "^10.2.1", "tmp": "^0.2.1", diff --git a/packages/types/package.json b/packages/types/package.json index 6d69d501134f..d5512763b197 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": { ".": { @@ -71,7 +71,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/ssz": "^0.15.1", - "@lodestar/params": "^1.20.1", + "@lodestar/params": "^1.20.2", "ethereum-cryptography": "^2.0.0" }, "keywords": [ diff --git a/packages/utils/package.json b/packages/utils/package.json index 8ad6f8b6b93b..e3ada3e2901b 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.20.1", + "version": "1.20.2", "type": "module", "exports": "./lib/index.js", "files": [ diff --git a/packages/validator/package.json b/packages/validator/package.json index c4929ffb6eca..5290698d4adb 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/validator", - "version": "1.20.1", + "version": "1.20.2", "description": "A Typescript implementation of the validator client", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -47,17 +47,17 @@ "dependencies": { "@chainsafe/bls": "7.1.3", "@chainsafe/ssz": "^0.15.1", - "@lodestar/api": "^1.20.1", - "@lodestar/config": "^1.20.1", - "@lodestar/db": "^1.20.1", - "@lodestar/params": "^1.20.1", - "@lodestar/state-transition": "^1.20.1", - "@lodestar/types": "^1.20.1", - "@lodestar/utils": "^1.20.1", + "@lodestar/api": "^1.20.2", + "@lodestar/config": "^1.20.2", + "@lodestar/db": "^1.20.2", + "@lodestar/params": "^1.20.2", + "@lodestar/state-transition": "^1.20.2", + "@lodestar/types": "^1.20.2", + "@lodestar/utils": "^1.20.2", "strict-event-emitter-types": "^2.0.0" }, "devDependencies": { - "@lodestar/test-utils": "^1.20.1", + "@lodestar/test-utils": "^1.20.2", "bigint-buffer": "^1.1.5", "rimraf": "^4.4.1" } From fb6bef857bb8f0535f150c29d6c9a9e1faa29999 Mon Sep 17 00:00:00 2001 From: g11tech Date: Fri, 19 Jul 2024 14:54:05 +0530 Subject: [PATCH 2/4] fix: fix the publish blinded block api parsing for optional header verison (#6966) --- packages/api/src/beacon/routes/beacon/block.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/packages/api/src/beacon/routes/beacon/block.ts b/packages/api/src/beacon/routes/beacon/block.ts index cbf206e51a4a..73680ac0afc2 100644 --- a/packages/api/src/beacon/routes/beacon/block.ts +++ b/packages/api/src/beacon/routes/beacon/block.ts @@ -444,7 +444,16 @@ export function getDefinitions(config: ChainForkConfig): RouteDefinitions { - const fork = toForkName(fromHeaders(headers, MetaHeader.Version)); + let fork: ForkName; + // As per spec, version header is optional for JSON requests + const versionHeader = fromHeaders(headers, MetaHeader.Version, false); + if (versionHeader !== undefined) { + fork = toForkName(versionHeader); + } else { + // Determine fork from slot in JSON payload + fork = config.getForkName((body as SignedBlindedBeaconBlock).message.slot); + } + return { signedBlindedBlock: getExecutionForkTypes(fork).SignedBlindedBeaconBlock.fromJson(body), }; From f20484bb4b6b5f3a27c624556d1e6fb9f448b969 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Thu, 18 Jul 2024 18:47:03 +0100 Subject: [PATCH 3/4] chore: pin nodejs version to 22.4 (#6964) * chore: pin nodejs version to 22.4 * More pinning --- .github/workflows/benchmark.yml | 2 +- .github/workflows/binaries.yml | 2 +- .github/workflows/docs-check.yml | 4 ++-- .github/workflows/docs.yml | 8 ++++---- .github/workflows/publish-dev.yml | 4 ++-- .github/workflows/publish-rc.yml | 2 +- .github/workflows/publish-stable.yml | 2 +- .github/workflows/test-sim-merge.yml | 2 +- .github/workflows/test-sim.yml | 12 ++++++------ .github/workflows/test.yml | 14 +++++++------- Dockerfile | 6 +++--- package.json | 2 +- 12 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index e515bef7f92a..671434fe19ad 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -33,7 +33,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 22.4 check-latest: true cache: yarn - name: Node.js version diff --git a/.github/workflows/binaries.yml b/.github/workflows/binaries.yml index 469b0803c378..722894424b91 100644 --- a/.github/workflows/binaries.yml +++ b/.github/workflows/binaries.yml @@ -42,7 +42,7 @@ jobs: sudo apt-get install -y build-essential python3 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - run: | mkdir -p dist yarn global add caxa@3.0.1 diff --git a/.github/workflows/docs-check.yml b/.github/workflows/docs-check.yml index 021e5019760d..bd7310995d62 100644 --- a/.github/workflows/docs-check.yml +++ b/.github/workflows/docs-check.yml @@ -15,8 +15,8 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 22 - cache: yarn + node-version: 22.4 + cache: yarn - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 63db55435edf..a4c0f18cdbe3 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -7,9 +7,9 @@ on: workflow_dispatch: inputs: ref: - description: 'Ref to deploy, defaults to `unstable`' + description: "Ref to deploy, defaults to `unstable`" required: false - default: 'unstable' + default: "unstable" type: string jobs: @@ -31,9 +31,9 @@ jobs: - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 22.4 check-latest: true - cache: yarn + cache: yarn - name: Node.js version id: node diff --git a/.github/workflows/publish-dev.yml b/.github/workflows/publish-dev.yml index da045764dd41..fb4197b75da9 100644 --- a/.github/workflows/publish-dev.yml +++ b/.github/workflows/publish-dev.yml @@ -20,10 +20,10 @@ jobs: fetch-depth: 0 - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 22.4 registry-url: "https://registry.npmjs.org" check-latest: true - cache: yarn + cache: yarn - name: Node.js version id: node run: echo "v8CppApiVersion=$(node --print "process.versions.modules")" >> $GITHUB_OUTPUT diff --git a/.github/workflows/publish-rc.yml b/.github/workflows/publish-rc.yml index 35831822bcdc..f8f3b21ff349 100644 --- a/.github/workflows/publish-rc.yml +++ b/.github/workflows/publish-rc.yml @@ -61,7 +61,7 @@ jobs: - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Generate changelog run: node scripts/generate_changelog.mjs ${{ needs.tag.outputs.prev_tag }} ${{ needs.tag.outputs.tag }} CHANGELOG.md diff --git a/.github/workflows/publish-stable.yml b/.github/workflows/publish-stable.yml index 741c060e4d20..9c41693f26f2 100644 --- a/.github/workflows/publish-stable.yml +++ b/.github/workflows/publish-stable.yml @@ -67,7 +67,7 @@ jobs: - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Generate changelog run: node scripts/generate_changelog.mjs ${{ needs.tag.outputs.prev_tag }} ${{ needs.tag.outputs.tag }} CHANGELOG.md diff --git a/.github/workflows/test-sim-merge.yml b/.github/workflows/test-sim-merge.yml index ad79bc2c0035..0042a9337bc3 100644 --- a/.github/workflows/test-sim-merge.yml +++ b/.github/workflows/test-sim-merge.yml @@ -30,7 +30,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-node@v4 with: - node-version: 22 + node-version: 22.4 check-latest: true cache: yarn - name: Node.js version diff --git a/.github/workflows/test-sim.yml b/.github/workflows/test-sim.yml index ff28149537d3..fbe2691da637 100644 --- a/.github/workflows/test-sim.yml +++ b/.github/workflows/test-sim.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 sim-test-multifork: name: Multifork sim test @@ -42,7 +42,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests @@ -71,7 +71,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests @@ -100,7 +100,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests @@ -129,7 +129,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests @@ -158,7 +158,7 @@ jobs: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" with: - node: 22 + node: 22.4 - name: Load env variables uses: ./.github/actions/dotenv - name: Download required docker images before running tests diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f55eb661629f..c801b0462d60 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -19,7 +19,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: # - Uses YAML anchors in the future - uses: actions/checkout@v4 @@ -42,7 +42,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" @@ -74,7 +74,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: - uses: actions/checkout@v4 @@ -95,7 +95,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" @@ -134,7 +134,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: # - Uses YAML anchors in the future - uses: actions/checkout@v4 @@ -171,7 +171,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: # - Uses YAML anchors in the future - uses: actions/checkout@v4 @@ -195,7 +195,7 @@ jobs: strategy: fail-fast: false matrix: - node: [22] + node: [22.4] steps: - uses: actions/checkout@v4 - uses: "./.github/actions/setup-and-build" diff --git a/Dockerfile b/Dockerfile index fdab1140af47..c65cac28d51d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # --platform=$BUILDPLATFORM is used build javascript source with host arch # Otherwise TS builds on emulated archs and can be extremely slow (+1h) -FROM --platform=${BUILDPLATFORM:-amd64} node:22-alpine as build_src +FROM --platform=${BUILDPLATFORM:-amd64} node:22.4-alpine as build_src ARG COMMIT WORKDIR /usr/app RUN apk update && apk add --no-cache g++ make python3 py3-setuptools && rm -rf /var/cache/apk/* @@ -21,7 +21,7 @@ RUN cd packages/cli && GIT_COMMIT=${COMMIT} yarn write-git-data # Copy built src + node_modules to build native packages for archs different than host. # Note: This step is redundant for the host arch -FROM node:22-alpine as build_deps +FROM node:22.4-alpine as build_deps WORKDIR /usr/app RUN apk update && apk add --no-cache g++ make python3 py3-setuptools && rm -rf /var/cache/apk/* @@ -35,7 +35,7 @@ RUN cd node_modules/classic-level && yarn rebuild # Copy built src + node_modules to a new layer to prune unnecessary fs # Previous layer weights 7.25GB, while this final 488MB (as of Oct 2020) -FROM node:22-alpine +FROM node:22.4-alpine WORKDIR /usr/app COPY --from=build_deps /usr/app . diff --git a/package.json b/package.json index e4464b3b5a9f..87ccd7e9f031 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "root", "private": true, "engines": { - "node": ">=20.1.0 <21 || >=22 <23" + "node": ">=20.1.0 <21 || >=22 <22.5" }, "packageManager": "yarn@1.22.22+sha256.c17d3797fb9a9115bf375e31bfd30058cac6bc9c3b8807a3d8cb2094794b51ca", "workspaces": [ From 810f4a5a66311bee5c62a4e5eabed37456d15591 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 15 Jul 2024 15:19:58 +0100 Subject: [PATCH 4/4] feat: add option to disable thread pool for keystore decryption (#6949) --- .../keymanager/decryptKeystoreDefinitions.ts | 69 ++++++++++++++----- packages/cli/src/cmds/validator/options.ts | 8 +++ .../cli/src/cmds/validator/signers/index.ts | 2 + .../decryptKeystoreDefinitions.test.ts | 23 ++++--- 4 files changed, 77 insertions(+), 25 deletions(-) diff --git a/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts b/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts index 2901fd6cdfb5..076da70adcdb 100644 --- a/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts +++ b/packages/cli/src/cmds/validator/keymanager/decryptKeystoreDefinitions.ts @@ -1,5 +1,7 @@ +import fs from "node:fs"; import path from "node:path"; import bls from "@chainsafe/bls"; +import {Keystore} from "@chainsafe/bls-keystore"; import {SignerLocal, SignerType} from "@lodestar/validator"; import {LogLevel, Logger} from "@lodestar/utils"; import {lockFilepath, unlockFilepath} from "../../../util/lockfile.js"; @@ -7,11 +9,13 @@ import {LocalKeystoreDefinition} from "./interface.js"; import {clearKeystoreCache, loadKeystoreCache, writeKeystoreCache} from "./keystoreCache.js"; import {DecryptKeystoresThreadPool} from "./decryptKeystores/index.js"; -type KeystoreDecryptOptions = { +export type KeystoreDecryptOptions = { ignoreLockFile?: boolean; onDecrypt?: (index: number) => void; // Try to use the cache file if it exists cacheFilePath?: string; + /** Use main thread to decrypt keystores */ + disableThreadPool?: boolean; logger: Pick; signal: AbortSignal; }; @@ -57,14 +61,50 @@ export async function decryptKeystoreDefinitions( const signers = new Array(keystoreCount); const passwords = new Array(keystoreCount); const errors: KeystoreDecryptError[] = []; - const decryptKeystores = new DecryptKeystoresThreadPool(keystoreCount, opts.signal); - for (const [index, definition] of keystoreDefinitions.entries()) { - lockKeystore(definition.keystorePath, opts); + if (!opts.disableThreadPool) { + const decryptKeystores = new DecryptKeystoresThreadPool(keystoreCount, opts.signal); + + for (const [index, definition] of keystoreDefinitions.entries()) { + lockKeystore(definition.keystorePath, opts); + + decryptKeystores.queue( + definition, + (secretKeyBytes: Uint8Array) => { + const signer: SignerLocal = { + type: SignerType.Local, + secretKey: bls.SecretKey.fromBytes(secretKeyBytes), + }; + + signers[index] = signer; + passwords[index] = definition.password; + + if (opts?.onDecrypt) { + opts?.onDecrypt(index); + } + }, + (error: Error) => { + // In-progress tasks can't be canceled, so there's a chance that multiple errors may be caught + // add to the list of errors + errors.push({keystoreFile: path.basename(definition.keystorePath), error}); + // cancel all pending tasks, no need to continue decrypting after we hit one error + decryptKeystores.cancel(); + } + ); + } + + await decryptKeystores.completed(); + } else { + // Decrypt keystores in main thread + for (const [index, definition] of keystoreDefinitions.entries()) { + lockKeystore(definition.keystorePath, opts); + + try { + const keystore = Keystore.parse(fs.readFileSync(definition.keystorePath, "utf8")); + + // Memory-hogging function + const secretKeyBytes = await keystore.decrypt(definition.password); - decryptKeystores.queue( - definition, - (secretKeyBytes: Uint8Array) => { const signer: SignerLocal = { type: SignerType.Local, secretKey: bls.SecretKey.fromBytes(secretKeyBytes), @@ -76,19 +116,14 @@ export async function decryptKeystoreDefinitions( if (opts?.onDecrypt) { opts?.onDecrypt(index); } - }, - (error: Error) => { - // In-progress tasks can't be canceled, so there's a chance that multiple errors may be caught - // add to the list of errors - errors.push({keystoreFile: path.basename(definition.keystorePath), error}); - // cancel all pending tasks, no need to continue decrypting after we hit one error - decryptKeystores.cancel(); + } catch (e) { + errors.push({keystoreFile: path.basename(definition.keystorePath), error: e as Error}); + // stop processing, no need to continue decrypting after we hit one error + break; } - ); + } } - await decryptKeystores.completed(); - if (errors.length > 0) { // If an error occurs, the program isn't going to be running, // so we should unlock all lockfiles we created diff --git a/packages/cli/src/cmds/validator/options.ts b/packages/cli/src/cmds/validator/options.ts index 08548edb1072..fc7cb197c5aa 100644 --- a/packages/cli/src/cmds/validator/options.ts +++ b/packages/cli/src/cmds/validator/options.ts @@ -55,6 +55,7 @@ export type IValidatorCliArgs = AccountValidatorArgs & importKeystores?: string[]; importKeystoresPassword?: string; + disableKeystoresThreadPool?: boolean; "http.requestWireFormat"?: string; "http.responseWireFormat"?: string; @@ -301,6 +302,13 @@ export const validatorOptions: CliCommandOptions = { type: "string", }, + disableKeystoresThreadPool: { + hidden: true, + description: + "Disable thread pool and instead use main thread to decrypt keystores. This can speed up decryption in testing environments like Kurtosis", + type: "boolean", + }, + doppelgangerProtection: { alias: ["doppelgangerProtectionEnabled"], description: "Enables Doppelganger protection", diff --git a/packages/cli/src/cmds/validator/signers/index.ts b/packages/cli/src/cmds/validator/signers/index.ts index be028461c0ee..95daf7e69b0d 100644 --- a/packages/cli/src/cmds/validator/signers/index.ts +++ b/packages/cli/src/cmds/validator/signers/index.ts @@ -99,6 +99,7 @@ export async function getSignersFromArgs( ignoreLockFile: args.force, onDecrypt: needle, cacheFilePath: path.join(accountPaths.cacheDir, "imported_keystores.cache"), + disableThreadPool: args["disableKeystoresThreadPool"], logger, signal, }); @@ -133,6 +134,7 @@ export async function getSignersFromArgs( ignoreLockFile: args.force, onDecrypt: needle, cacheFilePath: path.join(accountPaths.cacheDir, "local_keystores.cache"), + disableThreadPool: args["disableKeystoresThreadPool"], logger, signal, }); diff --git a/packages/cli/test/unit/validator/decryptKeystoreDefinitions.test.ts b/packages/cli/test/unit/validator/decryptKeystoreDefinitions.test.ts index 8f1a82a55c3e..10f6b34bd152 100644 --- a/packages/cli/test/unit/validator/decryptKeystoreDefinitions.test.ts +++ b/packages/cli/test/unit/validator/decryptKeystoreDefinitions.test.ts @@ -5,7 +5,10 @@ import {rimraf} from "rimraf"; import {getKeystoresStr} from "@lodestar/test-utils"; import {cachedSeckeysHex} from "../../utils/cachedKeys.js"; import {testFilesDir} from "../../utils.js"; -import {decryptKeystoreDefinitions} from "../../../src/cmds/validator/keymanager/decryptKeystoreDefinitions.js"; +import { + decryptKeystoreDefinitions, + KeystoreDecryptOptions, +} from "../../../src/cmds/validator/keymanager/decryptKeystoreDefinitions.js"; import {LocalKeystoreDefinition} from "../../../src/cmds/validator/keymanager/interface.js"; import {LockfileError, unlockFilepath} from "../../../src/util/lockfile.js"; @@ -56,16 +59,20 @@ describe("decryptKeystoreDefinitions", () => { } }); - testDecryptKeystoreDefinitions(cacheFilePath); + testDecryptKeystoreDefinitions({cacheFilePath}); }); describe("without keystore cache", () => { testDecryptKeystoreDefinitions(); }); - function testDecryptKeystoreDefinitions(cacheFilePath?: string): void { + describe("disabled thread pool", () => { + testDecryptKeystoreDefinitions({disableThreadPool: true}); + }); + + function testDecryptKeystoreDefinitions(opts?: Partial): void { it("decrypt keystores", async () => { - const signers = await decryptKeystoreDefinitions(definitions, {logger: console, signal, cacheFilePath}); + const signers = await decryptKeystoreDefinitions(definitions, {logger: console, signal, ...opts}); expect(signers.length).toBe(secretKeys.length); for (const signer of signers) { const hexSecret = signer.secretKey.toHex(); @@ -75,11 +82,11 @@ describe("decryptKeystoreDefinitions", () => { }); it("fail to decrypt keystores if lockfiles already exist", async () => { - await decryptKeystoreDefinitions(definitions, {logger: console, signal, cacheFilePath}); + await decryptKeystoreDefinitions(definitions, {logger: console, signal, ...opts}); // lockfiles should exist after the first run try { - await decryptKeystoreDefinitions(definitions, {logger: console, signal, cacheFilePath}); + await decryptKeystoreDefinitions(definitions, {logger: console, signal, ...opts}); expect.fail("Second decrypt should fail due to failure to get lockfile"); } catch (e) { expect((e as LockfileError).code).toBe("ELOCKED"); @@ -87,10 +94,10 @@ describe("decryptKeystoreDefinitions", () => { }); it("decrypt keystores if lockfiles already exist if ignoreLockFile=true", async () => { - await decryptKeystoreDefinitions(definitions, {logger: console, signal, cacheFilePath}); + await decryptKeystoreDefinitions(definitions, {logger: console, signal, ...opts}); // lockfiles should exist after the first run - await decryptKeystoreDefinitions(definitions, {logger: console, signal, cacheFilePath, ignoreLockFile: true}); + await decryptKeystoreDefinitions(definitions, {logger: console, signal, ...opts, ignoreLockFile: true}); }); } });