From 92ed3479a6ac59b75d100b9738ee437b2e80a748 Mon Sep 17 00:00:00 2001 From: Phil Ngo Date: Tue, 23 Jan 2024 11:06:39 -0500 Subject: [PATCH 1/8] v1.15.0 --- lerna.json | 2 +- packages/api/package.json | 10 +++++----- packages/beacon-node/package.json | 26 +++++++++++++------------- packages/cli/package.json | 26 +++++++++++++------------- packages/config/package.json | 6 +++--- packages/db/package.json | 8 ++++---- packages/flare/package.json | 14 +++++++------- packages/fork-choice/package.json | 12 ++++++------ packages/light-client/package.json | 14 +++++++------- packages/logger/package.json | 6 +++--- packages/params/package.json | 2 +- packages/prover/package.json | 18 +++++++++--------- packages/reqresp/package.json | 12 ++++++------ packages/spec-test-util/package.json | 8 ++++---- packages/state-transition/package.json | 10 +++++----- packages/test-utils/package.json | 6 +++--- packages/types/package.json | 4 ++-- packages/utils/package.json | 2 +- packages/validator/package.json | 18 +++++++++--------- 19 files changed, 102 insertions(+), 102 deletions(-) diff --git a/lerna.json b/lerna.json index 487caa95b0a2..2a9439d49715 100644 --- a/lerna.json +++ b/lerna.json @@ -4,7 +4,7 @@ ], "npmClient": "yarn", "useNx": true, - "version": "1.14.0", + "version": "1.15.0", "stream": true, "command": { "version": { diff --git a/packages/api/package.json b/packages/api/package.json index e1a52c47b489..533961953b9a 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.14.0", + "version": "1.15.0", "type": "module", "exports": { ".": { @@ -71,10 +71,10 @@ "dependencies": { "@chainsafe/persistent-merkle-tree": "^0.6.1", "@chainsafe/ssz": "^0.14.0", - "@lodestar/config": "^1.14.0", - "@lodestar/params": "^1.14.0", - "@lodestar/types": "^1.14.0", - "@lodestar/utils": "^1.14.0", + "@lodestar/config": "^1.15.0", + "@lodestar/params": "^1.15.0", + "@lodestar/types": "^1.15.0", + "@lodestar/utils": "^1.15.0", "eventsource": "^2.0.2", "qs": "^6.11.1" }, diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index 3ddde8540670..8ce625fb0d54 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.14.0", + "version": "1.15.0", "type": "module", "exports": { ".": { @@ -121,18 +121,18 @@ "@libp2p/peer-id-factory": "^4.0.3", "@libp2p/prometheus-metrics": "^3.0.10", "@libp2p/tcp": "9.0.10", - "@lodestar/api": "^1.14.0", - "@lodestar/config": "^1.14.0", - "@lodestar/db": "^1.14.0", - "@lodestar/fork-choice": "^1.14.0", - "@lodestar/light-client": "^1.14.0", - "@lodestar/logger": "^1.14.0", - "@lodestar/params": "^1.14.0", - "@lodestar/reqresp": "^1.14.0", - "@lodestar/state-transition": "^1.14.0", - "@lodestar/types": "^1.14.0", - "@lodestar/utils": "^1.14.0", - "@lodestar/validator": "^1.14.0", + "@lodestar/api": "^1.15.0", + "@lodestar/config": "^1.15.0", + "@lodestar/db": "^1.15.0", + "@lodestar/fork-choice": "^1.15.0", + "@lodestar/light-client": "^1.15.0", + "@lodestar/logger": "^1.15.0", + "@lodestar/params": "^1.15.0", + "@lodestar/reqresp": "^1.15.0", + "@lodestar/state-transition": "^1.15.0", + "@lodestar/types": "^1.15.0", + "@lodestar/utils": "^1.15.0", + "@lodestar/validator": "^1.15.0", "@multiformats/multiaddr": "^12.1.3", "@types/datastore-level": "^3.0.0", "buffer-xor": "^2.0.2", diff --git a/packages/cli/package.json b/packages/cli/package.json index 243dee688417..616dce9d4f68 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,6 +1,6 @@ { "name": "@chainsafe/lodestar", - "version": "1.14.0", + "version": "1.15.0", "description": "Command line interface for lodestar", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -66,17 +66,17 @@ "@libp2p/crypto": "^3.0.4", "@libp2p/peer-id": "^4.0.4", "@libp2p/peer-id-factory": "^4.0.3", - "@lodestar/api": "^1.14.0", - "@lodestar/beacon-node": "^1.14.0", - "@lodestar/config": "^1.14.0", - "@lodestar/db": "^1.14.0", - "@lodestar/light-client": "^1.14.0", - "@lodestar/logger": "^1.14.0", - "@lodestar/params": "^1.14.0", - "@lodestar/state-transition": "^1.14.0", - "@lodestar/types": "^1.14.0", - "@lodestar/utils": "^1.14.0", - "@lodestar/validator": "^1.14.0", + "@lodestar/api": "^1.15.0", + "@lodestar/beacon-node": "^1.15.0", + "@lodestar/config": "^1.15.0", + "@lodestar/db": "^1.15.0", + "@lodestar/light-client": "^1.15.0", + "@lodestar/logger": "^1.15.0", + "@lodestar/params": "^1.15.0", + "@lodestar/state-transition": "^1.15.0", + "@lodestar/types": "^1.15.0", + "@lodestar/utils": "^1.15.0", + "@lodestar/validator": "^1.15.0", "@multiformats/multiaddr": "^12.1.3", "@types/lockfile": "^1.0.2", "bip39": "^3.1.0", @@ -97,7 +97,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.14.0", + "@lodestar/test-utils": "^1.15.0", "@types/debug": "^4.1.7", "@types/expand-tilde": "^2.0.0", "@types/got": "^9.6.12", diff --git a/packages/config/package.json b/packages/config/package.json index 3b0a649e78c5..a36da8c6240a 100644 --- a/packages/config/package.json +++ b/packages/config/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/config", - "version": "1.14.0", + "version": "1.15.0", "description": "Chain configuration required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -65,7 +65,7 @@ ], "dependencies": { "@chainsafe/ssz": "^0.14.0", - "@lodestar/params": "^1.14.0", - "@lodestar/types": "^1.14.0" + "@lodestar/params": "^1.15.0", + "@lodestar/types": "^1.15.0" } } diff --git a/packages/db/package.json b/packages/db/package.json index d99b21d80b9b..f11e5b48995c 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/db", - "version": "1.14.0", + "version": "1.15.0", "description": "DB modules of Lodestar", "author": "ChainSafe Systems", "homepage": "https://github.com/ChainSafe/lodestar#readme", @@ -38,13 +38,13 @@ }, "dependencies": { "@chainsafe/ssz": "^0.14.0", - "@lodestar/config": "^1.14.0", - "@lodestar/utils": "^1.14.0", + "@lodestar/config": "^1.15.0", + "@lodestar/utils": "^1.15.0", "@types/levelup": "^4.3.3", "it-all": "^3.0.4", "level": "^8.0.0" }, "devDependencies": { - "@lodestar/logger": "^1.14.0" + "@lodestar/logger": "^1.15.0" } } diff --git a/packages/flare/package.json b/packages/flare/package.json index 7417cf93c63d..93973110ab8a 100644 --- a/packages/flare/package.json +++ b/packages/flare/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/flare", - "version": "1.14.0", + "version": "1.15.0", "description": "Beacon chain debugging tool", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -60,12 +60,12 @@ "dependencies": { "@chainsafe/bls": "7.1.1", "@chainsafe/bls-keygen": "^0.3.0", - "@lodestar/api": "^1.14.0", - "@lodestar/config": "^1.14.0", - "@lodestar/params": "^1.14.0", - "@lodestar/state-transition": "^1.14.0", - "@lodestar/types": "^1.14.0", - "@lodestar/utils": "^1.14.0", + "@lodestar/api": "^1.15.0", + "@lodestar/config": "^1.15.0", + "@lodestar/params": "^1.15.0", + "@lodestar/state-transition": "^1.15.0", + "@lodestar/types": "^1.15.0", + "@lodestar/utils": "^1.15.0", "source-map-support": "^0.5.21", "yargs": "^17.7.1" }, diff --git a/packages/fork-choice/package.json b/packages/fork-choice/package.json index 495c67a8e80b..606dec811bbc 100644 --- a/packages/fork-choice/package.json +++ b/packages/fork-choice/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.14.0", + "version": "1.15.0", "type": "module", "exports": "./lib/index.js", "types": "./lib/index.d.ts", @@ -39,11 +39,11 @@ }, "dependencies": { "@chainsafe/ssz": "^0.14.0", - "@lodestar/config": "^1.14.0", - "@lodestar/params": "^1.14.0", - "@lodestar/state-transition": "^1.14.0", - "@lodestar/types": "^1.14.0", - "@lodestar/utils": "^1.14.0" + "@lodestar/config": "^1.15.0", + "@lodestar/params": "^1.15.0", + "@lodestar/state-transition": "^1.15.0", + "@lodestar/types": "^1.15.0", + "@lodestar/utils": "^1.15.0" }, "keywords": [ "ethereum", diff --git a/packages/light-client/package.json b/packages/light-client/package.json index dee231409730..456600cbd0bc 100644 --- a/packages/light-client/package.json +++ b/packages/light-client/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.14.0", + "version": "1.15.0", "type": "module", "exports": { ".": { @@ -70,12 +70,12 @@ "@chainsafe/bls": "7.1.1", "@chainsafe/persistent-merkle-tree": "^0.6.1", "@chainsafe/ssz": "^0.14.0", - "@lodestar/api": "^1.14.0", - "@lodestar/config": "^1.14.0", - "@lodestar/params": "^1.14.0", - "@lodestar/state-transition": "^1.14.0", - "@lodestar/types": "^1.14.0", - "@lodestar/utils": "^1.14.0", + "@lodestar/api": "^1.15.0", + "@lodestar/config": "^1.15.0", + "@lodestar/params": "^1.15.0", + "@lodestar/state-transition": "^1.15.0", + "@lodestar/types": "^1.15.0", + "@lodestar/utils": "^1.15.0", "mitt": "^3.0.0", "strict-event-emitter-types": "^2.0.0" }, diff --git a/packages/logger/package.json b/packages/logger/package.json index 2f450ee5be12..f1a5ddad1a06 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.14.0", + "version": "1.15.0", "type": "module", "exports": { ".": { @@ -66,14 +66,14 @@ }, "types": "lib/index.d.ts", "dependencies": { - "@lodestar/utils": "^1.14.0", + "@lodestar/utils": "^1.15.0", "winston": "^3.8.2", "winston-daily-rotate-file": "^4.7.1", "winston-transport": "^4.5.0" }, "devDependencies": { "@chainsafe/threads": "^1.11.1", - "@lodestar/test-utils": "^1.14.0", + "@lodestar/test-utils": "^1.15.0", "@types/triple-beam": "^1.3.2", "rimraf": "^4.4.1", "triple-beam": "^1.3.0" diff --git a/packages/params/package.json b/packages/params/package.json index 2586c73566aa..d41dae06816e 100644 --- a/packages/params/package.json +++ b/packages/params/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/params", - "version": "1.14.0", + "version": "1.15.0", "description": "Chain parameters required for lodestar", "author": "ChainSafe Systems", "license": "Apache-2.0", diff --git a/packages/prover/package.json b/packages/prover/package.json index 05f7c33579b4..600f4b4d4d75 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.14.0", + "version": "1.15.0", "type": "module", "exports": { ".": { @@ -72,13 +72,13 @@ "@ethereumjs/tx": "^4.1.2", "@ethereumjs/util": "^8.0.6", "@ethereumjs/vm": "^6.4.2", - "@lodestar/api": "^1.14.0", - "@lodestar/config": "^1.14.0", - "@lodestar/light-client": "^1.14.0", - "@lodestar/logger": "^1.14.0", - "@lodestar/params": "^1.14.0", - "@lodestar/types": "^1.14.0", - "@lodestar/utils": "^1.14.0", + "@lodestar/api": "^1.15.0", + "@lodestar/config": "^1.15.0", + "@lodestar/light-client": "^1.15.0", + "@lodestar/logger": "^1.15.0", + "@lodestar/params": "^1.15.0", + "@lodestar/types": "^1.15.0", + "@lodestar/utils": "^1.15.0", "ethereum-cryptography": "^1.2.0", "find-up": "^6.3.0", "http-proxy": "^1.18.1", @@ -87,7 +87,7 @@ "yargs": "^17.7.1" }, "devDependencies": { - "@lodestar/test-utils": "^1.14.0", + "@lodestar/test-utils": "^1.15.0", "@types/http-proxy": "^1.17.10", "@types/yargs": "^17.0.24", "axios": "^1.3.4", diff --git a/packages/reqresp/package.json b/packages/reqresp/package.json index c19af565c397..3856177449d3 100644 --- a/packages/reqresp/package.json +++ b/packages/reqresp/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.14.0", + "version": "1.15.0", "type": "module", "exports": { ".": { @@ -56,9 +56,9 @@ "dependencies": { "@chainsafe/fast-crc32c": "^4.1.1", "@libp2p/interface": "^1.1.1", - "@lodestar/config": "^1.14.0", - "@lodestar/params": "^1.14.0", - "@lodestar/utils": "^1.14.0", + "@lodestar/config": "^1.15.0", + "@lodestar/params": "^1.15.0", + "@lodestar/utils": "^1.15.0", "it-all": "^3.0.4", "it-pipe": "^3.0.1", "snappy": "^7.2.2", @@ -67,8 +67,8 @@ "uint8arraylist": "^2.4.7" }, "devDependencies": { - "@lodestar/logger": "^1.14.0", - "@lodestar/types": "^1.14.0", + "@lodestar/logger": "^1.15.0", + "@lodestar/types": "^1.15.0", "libp2p": "1.1.1" }, "peerDependencies": { diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json index 899d6438d939..f8d953e408fd 100644 --- a/packages/spec-test-util/package.json +++ b/packages/spec-test-util/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/spec-test-util", - "version": "1.14.0", + "version": "1.15.0", "description": "Spec test suite generator from yaml test files", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -62,13 +62,13 @@ "blockchain" ], "dependencies": { - "@lodestar/utils": "^1.14.0", + "@lodestar/utils": "^1.15.0", "async-retry": "^1.3.3", "axios": "^1.3.4", - "vitest": "^1.2.1", "rimraf": "^4.4.1", "snappyjs": "^0.7.0", - "tar": "^6.1.13" + "tar": "^6.1.13", + "vitest": "^1.2.1" }, "devDependencies": { "@types/async-retry": "1.4.3", diff --git a/packages/state-transition/package.json b/packages/state-transition/package.json index cbde9b062177..8a1aea19f140 100644 --- a/packages/state-transition/package.json +++ b/packages/state-transition/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.14.0", + "version": "1.15.0", "type": "module", "exports": { ".": { @@ -63,10 +63,10 @@ "@chainsafe/persistent-merkle-tree": "^0.6.1", "@chainsafe/persistent-ts": "^0.19.1", "@chainsafe/ssz": "^0.14.0", - "@lodestar/config": "^1.14.0", - "@lodestar/params": "^1.14.0", - "@lodestar/types": "^1.14.0", - "@lodestar/utils": "^1.14.0", + "@lodestar/config": "^1.15.0", + "@lodestar/params": "^1.15.0", + "@lodestar/types": "^1.15.0", + "@lodestar/utils": "^1.15.0", "bigint-buffer": "^1.1.5", "buffer-xor": "^2.0.2" }, diff --git a/packages/test-utils/package.json b/packages/test-utils/package.json index d4a1df589e8f..038cd743bdd7 100644 --- a/packages/test-utils/package.json +++ b/packages/test-utils/package.json @@ -1,7 +1,7 @@ { "name": "@lodestar/test-utils", "private": true, - "version": "1.14.0", + "version": "1.15.0", "description": "Test utilities reused across other packages", "author": "ChainSafe Systems", "license": "Apache-2.0", @@ -60,8 +60,8 @@ "dependencies": { "@chainsafe/bls": "7.1.1", "@chainsafe/bls-keystore": "^3.0.0", - "@lodestar/params": "^1.14.0", - "@lodestar/utils": "^1.14.0", + "@lodestar/params": "^1.15.0", + "@lodestar/utils": "^1.15.0", "axios": "^1.3.4", "testcontainers": "^10.2.1", "tmp": "^0.2.1", diff --git a/packages/types/package.json b/packages/types/package.json index b51acbef6801..1e9b811becfa 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.14.0", + "version": "1.15.0", "type": "module", "exports": { ".": { @@ -73,7 +73,7 @@ "types": "lib/index.d.ts", "dependencies": { "@chainsafe/ssz": "^0.14.0", - "@lodestar/params": "^1.14.0" + "@lodestar/params": "^1.15.0" }, "keywords": [ "ethereum", diff --git a/packages/utils/package.json b/packages/utils/package.json index fcc0f5ac36c1..fa1dd4fdd829 100644 --- a/packages/utils/package.json +++ b/packages/utils/package.json @@ -11,7 +11,7 @@ "bugs": { "url": "https://github.com/ChainSafe/lodestar/issues" }, - "version": "1.14.0", + "version": "1.15.0", "type": "module", "exports": "./lib/index.js", "files": [ diff --git a/packages/validator/package.json b/packages/validator/package.json index 92c3e92d04a1..020792107c3b 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -1,6 +1,6 @@ { "name": "@lodestar/validator", - "version": "1.14.0", + "version": "1.15.0", "description": "A Typescript implementation of the validator client", "author": "ChainSafe Systems", "license": "LGPL-3.0", @@ -49,18 +49,18 @@ "dependencies": { "@chainsafe/bls": "7.1.1", "@chainsafe/ssz": "^0.14.0", - "@lodestar/api": "^1.14.0", - "@lodestar/config": "^1.14.0", - "@lodestar/db": "^1.14.0", - "@lodestar/params": "^1.14.0", - "@lodestar/state-transition": "^1.14.0", - "@lodestar/types": "^1.14.0", - "@lodestar/utils": "^1.14.0", + "@lodestar/api": "^1.15.0", + "@lodestar/config": "^1.15.0", + "@lodestar/db": "^1.15.0", + "@lodestar/params": "^1.15.0", + "@lodestar/state-transition": "^1.15.0", + "@lodestar/types": "^1.15.0", + "@lodestar/utils": "^1.15.0", "bigint-buffer": "^1.1.5", "strict-event-emitter-types": "^2.0.0" }, "devDependencies": { - "@lodestar/test-utils": "^1.14.0", + "@lodestar/test-utils": "^1.15.0", "bigint-buffer": "^1.1.5", "rimraf": "^4.4.1" } From 347c95fb0da219b5a4d551b7ce014f2ceda7ae48 Mon Sep 17 00:00:00 2001 From: g11tech Date: Fri, 26 Jan 2024 22:28:28 +0530 Subject: [PATCH 2/8] feat: allow builder boost factor to be configured via proposer config file (#6357) * feat: allow builder boost factor to be configured via proposer config file * reuse parseBuilderBoostFactor fn * re-shuffle fns and add builder selection validation --- packages/cli/src/cmds/validator/handler.ts | 29 +------------- packages/cli/src/util/proposerConfig.ts | 38 ++++++++++++++++++- .../validator/parseProposerConfig.test.ts | 3 ++ .../validator/proposerConfigs/validData.yaml | 2 + 4 files changed, 42 insertions(+), 30 deletions(-) diff --git a/packages/cli/src/cmds/validator/handler.ts b/packages/cli/src/cmds/validator/handler.ts index 91789c99ee3b..a98eb0755ad0 100644 --- a/packages/cli/src/cmds/validator/handler.ts +++ b/packages/cli/src/cmds/validator/handler.ts @@ -22,6 +22,7 @@ import {GlobalArgs} from "../../options/index.js"; import {YargsError, cleanOldLogFiles, getDefaultGraffiti, mkdir, parseLoggerArgs} from "../../util/index.js"; import {onGracefulShutdown, parseFeeRecipient, parseProposerConfig} from "../../util/index.js"; import {getVersionData} from "../../util/version.js"; +import {parseBuilderSelection, parseBuilderBoostFactor} from "../../util/proposerConfig.js"; import {getAccountPaths, getValidatorPaths} from "./paths.js"; import {IValidatorCliArgs, validatorMetricsDefaultOptions, validatorMonitoringDefaultOptions} from "./options.js"; import {getSignersFromArgs} from "./signers/index.js"; @@ -254,24 +255,6 @@ function getProposerConfigFromArgs( return valProposerConfig; } -function parseBuilderSelection(builderSelection?: string): routes.validator.BuilderSelection | undefined { - if (builderSelection) { - switch (builderSelection) { - case "maxprofit": - break; - case "builderalways": - break; - case "builderonly": - break; - case "executiononly": - break; - default: - throw new YargsError("Invalid input for builder selection, check help"); - } - } - return builderSelection as routes.validator.BuilderSelection; -} - function parseBroadcastValidation(broadcastValidation?: string): routes.beacon.BroadcastValidation | undefined { if (broadcastValidation) { switch (broadcastValidation) { @@ -286,13 +269,3 @@ function parseBroadcastValidation(broadcastValidation?: string): routes.beacon.B return broadcastValidation as routes.beacon.BroadcastValidation; } - -function parseBuilderBoostFactor(boostFactor?: string): bigint | undefined { - if (boostFactor === undefined) return; - - if (!/^\d+$/.test(boostFactor)) { - throw new YargsError("Invalid input for builder boost factor, must be a valid number without decimals"); - } - - return BigInt(boostFactor); -} diff --git a/packages/cli/src/util/proposerConfig.ts b/packages/cli/src/util/proposerConfig.ts index 2f3c71236255..661c81ee63af 100644 --- a/packages/cli/src/util/proposerConfig.ts +++ b/packages/cli/src/util/proposerConfig.ts @@ -8,6 +8,7 @@ import {routes} from "@lodestar/api"; import {parseFeeRecipient} from "./feeRecipient.js"; import {readFile} from "./file.js"; +import {YargsError} from "./index.js"; type ProposerConfig = ValidatorProposerConfig["defaultConfig"]; @@ -20,6 +21,7 @@ type ProposerConfigFileSection = { // for js-yaml gas_limit?: number; selection?: routes.validator.BuilderSelection; + boost_factor?: bigint; }; }; @@ -57,7 +59,7 @@ function parseProposerConfigSection( overrideConfig?: ProposerConfig ): ProposerConfig { const {graffiti, strict_fee_recipient_check, fee_recipient, builder} = proposerFileSection; - const {gas_limit, selection: builderSelection} = builder || {}; + const {gas_limit, selection: builderSelection, boost_factor} = builder || {}; if (graffiti !== undefined && typeof graffiti !== "string") { throw Error("graffiti is not 'string"); @@ -79,6 +81,9 @@ function parseProposerConfigSection( throw Error("(Number.isNaN(Number(gas_limit)) 2"); } } + if (boost_factor !== undefined && typeof boost_factor !== "string") { + throw Error("boost_factor is not 'string"); + } return { graffiti: overrideConfig?.graffiti ?? graffiti, @@ -88,7 +93,8 @@ function parseProposerConfigSection( feeRecipient: overrideConfig?.feeRecipient ?? (fee_recipient ? parseFeeRecipient(fee_recipient) : undefined), builder: { gasLimit: overrideConfig?.builder?.gasLimit ?? (gas_limit !== undefined ? Number(gas_limit) : undefined), - selection: overrideConfig?.builder?.selection ?? builderSelection, + selection: overrideConfig?.builder?.selection ?? parseBuilderSelection(builderSelection), + boostFactor: overrideConfig?.builder?.boostFactor ?? parseBuilderBoostFactor(boost_factor), }, }; } @@ -98,3 +104,31 @@ export function readProposerConfigDir(filepath: string, filename: string): Propo const proposerConfigJSON = JSON.parse(proposerConfigStr) as ProposerConfigFileSection; return proposerConfigJSON; } + +export function parseBuilderSelection(builderSelection?: string): routes.validator.BuilderSelection | undefined { + if (builderSelection) { + switch (builderSelection) { + case "maxprofit": + break; + case "builderalways": + break; + case "builderonly": + break; + case "executiononly": + break; + default: + throw new YargsError("Invalid input for builder selection, check help"); + } + } + return builderSelection as routes.validator.BuilderSelection; +} + +export function parseBuilderBoostFactor(boostFactor?: string): bigint | undefined { + if (boostFactor === undefined) return; + + if (!/^\d+$/.test(boostFactor)) { + throw new YargsError("Invalid input for builder boost factor, must be a valid number without decimals"); + } + + return BigInt(boostFactor); +} diff --git a/packages/cli/test/unit/validator/parseProposerConfig.test.ts b/packages/cli/test/unit/validator/parseProposerConfig.test.ts index fcb6933f035b..993d9640e47c 100644 --- a/packages/cli/test/unit/validator/parseProposerConfig.test.ts +++ b/packages/cli/test/unit/validator/parseProposerConfig.test.ts @@ -17,6 +17,7 @@ const testValue = { builder: { gasLimit: 30000000, selection: undefined, + boostFactor: undefined, }, }, "0xa4855c83d868f772a579133d9f23818008417b743e8447e235d8eb78b1d8f8a9f63f98c551beb7de254400f89592314d": { @@ -26,6 +27,7 @@ const testValue = { builder: { gasLimit: 35000000, selection: routes.validator.BuilderSelection.MaxProfit, + boostFactor: BigInt(18446744073709551616), }, }, }, @@ -36,6 +38,7 @@ const testValue = { builder: { gasLimit: 30000000, selection: routes.validator.BuilderSelection.BuilderAlways, + boostFactor: BigInt(100), }, }, }; diff --git a/packages/cli/test/unit/validator/proposerConfigs/validData.yaml b/packages/cli/test/unit/validator/proposerConfigs/validData.yaml index 6b7e7074b118..5ea7e1bebea7 100644 --- a/packages/cli/test/unit/validator/proposerConfigs/validData.yaml +++ b/packages/cli/test/unit/validator/proposerConfigs/validData.yaml @@ -10,6 +10,7 @@ proposer_config: builder: gas_limit: "35000000" selection: "maxprofit" + boost_factor: "18446744073709551616" default_config: graffiti: 'default graffiti' strict_fee_recipient_check: "true" @@ -17,3 +18,4 @@ default_config: builder: gas_limit: "30000000" selection: "builderalways" + boost_factor: "100" From 6adbd27c85f6fbb3a2a17ef4552274e985319c02 Mon Sep 17 00:00:00 2001 From: g11tech Date: Sat, 27 Jan 2024 22:37:49 +0530 Subject: [PATCH 3/8] fix: ignore forkchoice invalidations if latestValidHash not found (#6361) * fix: ignore forkchoice invalidations if latestValidHash not found * rename for better understanding * update the lvh search start index * apply feedback --- .../blocks/verifyBlocksExecutionPayloads.ts | 4 +- .../fork-choice/src/protoArray/interface.ts | 2 +- .../fork-choice/src/protoArray/protoArray.ts | 45 ++++++++++--------- .../protoArray/executionStatusUpdates.test.ts | 10 ++--- 4 files changed, 31 insertions(+), 30 deletions(-) diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts index 5dbe104c9541..91242d879f85 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksExecutionPayloads.ts @@ -319,7 +319,7 @@ export async function verifyBlockExecutionPayload( const lvhResponse = { executionStatus, latestValidExecHash: execResult.latestValidHash, - invalidateFromBlockHash: toHexString(block.message.parentRoot), + invalidateFromParentBlockRoot: toHexString(block.message.parentRoot), }; const execError = new BlockError(block, { code: BlockErrorCode.EXECUTION_ENGINE_ERROR, @@ -416,7 +416,7 @@ function getSegmentErrorResponse( invalidSegmentLVH = { executionStatus: ExecutionStatus.Invalid, latestValidExecHash: lvhResponse.latestValidExecHash, - invalidateFromBlockHash: parentBlock.blockRoot, + invalidateFromParentBlockRoot: parentBlock.blockRoot, }; } } diff --git a/packages/fork-choice/src/protoArray/interface.ts b/packages/fork-choice/src/protoArray/interface.ts index 1910ad2b4206..003a3c8f9f1e 100644 --- a/packages/fork-choice/src/protoArray/interface.ts +++ b/packages/fork-choice/src/protoArray/interface.ts @@ -29,7 +29,7 @@ export type LVHValidResponse = { export type LVHInvalidResponse = { executionStatus: ExecutionStatus.Invalid; latestValidExecHash: RootHex | null; - invalidateFromBlockHash: RootHex; + invalidateFromParentBlockRoot: RootHex; }; export type LVHExecResponse = LVHValidResponse | LVHInvalidResponse; diff --git a/packages/fork-choice/src/protoArray/protoArray.ts b/packages/fork-choice/src/protoArray/protoArray.ts index 62f50b03771c..eaa86b2f0ee1 100644 --- a/packages/fork-choice/src/protoArray/protoArray.ts +++ b/packages/fork-choice/src/protoArray/protoArray.ts @@ -279,32 +279,33 @@ export class ProtoArray { // Mark chain ii) as Invalid if LVH is found and non null, else only invalidate invalid_payload // if its in fcU. // - const {invalidateFromBlockHash, latestValidExecHash} = execResponse; - const invalidateFromIndex = this.indices.get(invalidateFromBlockHash); - if (invalidateFromIndex === undefined) { - throw Error(`Unable to find invalidateFromBlockHash=${invalidateFromBlockHash} in forkChoice`); + const {invalidateFromParentBlockRoot, latestValidExecHash} = execResponse; + const invalidateFromParentIndex = this.indices.get(invalidateFromParentBlockRoot); + if (invalidateFromParentIndex === undefined) { + throw Error(`Unable to find invalidateFromParentBlockRoot=${invalidateFromParentBlockRoot} in forkChoice`); } const latestValidHashIndex = - latestValidExecHash !== null ? this.getNodeIndexFromLVH(latestValidExecHash, invalidateFromIndex) : null; + latestValidExecHash !== null ? this.getNodeIndexFromLVH(latestValidExecHash, invalidateFromParentIndex) : null; if (latestValidHashIndex === null) { /** - * If the LVH is null or not found, represented with latestValidHashIndex=undefined, - * then just invalidate the invalid_payload and bug out. + * The LVH (latest valid hash) is null or not found. * - * Ideally in not found scenario we should invalidate the entire chain upwards, but - * it is possible (and observed in the testnets) that the EL was + * The spec gives an allowance for the EL being able to return a nullish LVH if it could not + * "determine" one. There are two interpretations: * - * i) buggy: that the LVH was not really the parent of the invalid block, but on - * some side chain - * ii) lazy: that invalidation was result of simple check and the EL just - * responded with a bogus LVH + * - "the LVH is unknown" - simply throw and move on. We can't determine which chain to invalidate + * since we don't know which ancestor is valid. * - * So we will just invalidate the current payload and let future responses take care - * to be as robust as possible. + * - "the LVH doesn't exist" - this means that the entire ancestor chain is invalid, and should + * be marked as such. + * + * The more robust approach is to treat nullish LVH as "the LVH is unknown" rather than + * "the LVH doesn't exist". The alternative means that we will poison a valid chain when the + * EL is lazy (or buggy) with its LVH response. */ - this.invalidateNodeByIndex(invalidateFromIndex); + throw Error(`Unable to find latestValidExecHash=${latestValidExecHash} in the forkchoice`); } else { - this.propagateInValidExecutionStatusByIndex(invalidateFromIndex, latestValidHashIndex, currentSlot); + this.propagateInValidExecutionStatusByIndex(invalidateFromParentIndex, latestValidHashIndex, currentSlot); } } } @@ -333,12 +334,12 @@ export class ProtoArray { */ private propagateInValidExecutionStatusByIndex( - invalidateFromIndex: number, + invalidateFromParentIndex: number, latestValidHashIndex: number, currentSlot: Slot ): void { - // Pass 1: mark invalidateFromIndex and its parents invalid - let invalidateIndex: number | undefined = invalidateFromIndex; + // Pass 1: mark invalidateFromParentIndex and its parents invalid + let invalidateIndex: number | undefined = invalidateFromParentIndex; while (invalidateIndex !== undefined && invalidateIndex > latestValidHashIndex) { const invalidNode = this.invalidateNodeByIndex(invalidateIndex); invalidateIndex = invalidNode.parent; @@ -368,8 +369,8 @@ export class ProtoArray { }); } - private getNodeIndexFromLVH(latestValidExecHash: RootHex, ancestorOfIndex: number): number | null { - let nodeIndex = this.nodes[ancestorOfIndex].parent; + private getNodeIndexFromLVH(latestValidExecHash: RootHex, ancestorFromIndex: number): number | null { + let nodeIndex: number | undefined = ancestorFromIndex; while (nodeIndex !== undefined && nodeIndex >= 0) { const node = this.getNodeFromIndex(nodeIndex); if ( diff --git a/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts b/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts index 94e5cd3ac9a0..e6916f24800f 100644 --- a/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts +++ b/packages/fork-choice/test/unit/protoArray/executionStatusUpdates.test.ts @@ -149,7 +149,7 @@ describe("executionStatus / normal updates", () => { { executionStatus: ExecutionStatus.Invalid, latestValidExecHash: "2C", - invalidateFromBlockHash: "3C", + invalidateFromParentBlockRoot: "3C", }, 3 ); @@ -212,7 +212,7 @@ describe("executionStatus / normal updates", () => { { executionStatus: ExecutionStatus.Invalid, latestValidExecHash: "1A", - invalidateFromBlockHash: "3A", + invalidateFromParentBlockRoot: "3A", }, 3 ); @@ -259,7 +259,7 @@ describe("executionStatus / invalidate all postmerge chain", () => { { executionStatus: ExecutionStatus.Invalid, latestValidExecHash: "0x0000000000000000000000000000000000000000000000000000000000000000", - invalidateFromBlockHash: "3B", + invalidateFromParentBlockRoot: "3B", }, 3 ); @@ -336,7 +336,7 @@ describe("executionStatus / poision forkchoice if we invalidate previous valid", { executionStatus: ExecutionStatus.Invalid, latestValidExecHash: "0x0000000000000000000000000000000000000000000000000000000000000000", - invalidateFromBlockHash: "3A", + invalidateFromParentBlockRoot: "3A", }, 3 ) @@ -373,7 +373,7 @@ describe("executionStatus / poision forkchoice if we validate previous invalid", { executionStatus: ExecutionStatus.Invalid, latestValidExecHash: "0x0000000000000000000000000000000000000000000000000000000000000000", - invalidateFromBlockHash: "3B", + invalidateFromParentBlockRoot: "3B", }, 3 ); From 291e17867becb3884fe502542bcede5bae1fc5b1 Mon Sep 17 00:00:00 2001 From: Matthew Keil Date: Mon, 29 Jan 2024 01:55:48 -0500 Subject: [PATCH 4/8] feat: add down-scoring for metadata ssz size error (#6366) --- packages/beacon-node/src/network/reqresp/score.ts | 1 + packages/reqresp/src/request/errors.ts | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/beacon-node/src/network/reqresp/score.ts b/packages/beacon-node/src/network/reqresp/score.ts index 6aca573a9ca9..c74b645c9909 100644 --- a/packages/beacon-node/src/network/reqresp/score.ts +++ b/packages/beacon-node/src/network/reqresp/score.ts @@ -24,6 +24,7 @@ export function onOutgoingReqRespError(e: RequestError, method: ReqRespMethod): switch (e.type.code) { case RequestErrorCode.INVALID_REQUEST: case RequestErrorCode.INVALID_RESPONSE_SSZ: + case RequestErrorCode.SSZ_OVER_MAX_SIZE: return PeerAction.LowToleranceError; case RequestErrorCode.SERVER_ERROR: diff --git a/packages/reqresp/src/request/errors.ts b/packages/reqresp/src/request/errors.ts index ccc3c12f97a1..0078ad5d2ecf 100644 --- a/packages/reqresp/src/request/errors.ts +++ b/packages/reqresp/src/request/errors.ts @@ -31,6 +31,8 @@ export enum RequestErrorCode { RESP_TIMEOUT = "REQUEST_ERROR_RESP_TIMEOUT", /** Request rate limited */ REQUEST_RATE_LIMITED = "REQUEST_ERROR_RATE_LIMITED", + /** */ + SSZ_OVER_MAX_SIZE = "SSZ_SNAPPY_ERROR_OVER_SSZ_MAX_SIZE", } type RequestErrorType = @@ -47,7 +49,8 @@ type RequestErrorType = | {code: RequestErrorCode.EMPTY_RESPONSE} | {code: RequestErrorCode.TTFB_TIMEOUT} | {code: RequestErrorCode.RESP_TIMEOUT} - | {code: RequestErrorCode.REQUEST_RATE_LIMITED}; + | {code: RequestErrorCode.REQUEST_RATE_LIMITED} + | {code: RequestErrorCode.SSZ_OVER_MAX_SIZE}; export const REQUEST_ERROR_CLASS_NAME = "RequestError"; From cbf349c73d7730c11d7295c27a61fbf36225df52 Mon Sep 17 00:00:00 2001 From: Julien Date: Mon, 29 Jan 2024 00:53:35 -0800 Subject: [PATCH 5/8] feat: blob sidecars can be filtered by indices (#6337) * feat: blob sidecars can be filtered by indices * fix: properly filter blobs * fix: type indices as number * chore: remove use of set * fix: make stringify more robust * Update packages/beacon-node/src/api/impl/beacon/blocks/index.ts Co-authored-by: Nico Flaig * fix: cleanup * fix: remove useless types --------- Co-authored-by: Nico Flaig --- .../api/src/beacon/routes/beacon/block.ts | 19 ++++++++++++++++--- .../api/test/unit/beacon/oapiSpec.test.ts | 6 ------ .../api/test/unit/beacon/testData/beacon.ts | 2 +- packages/api/test/utils/checkAgainstSpec.ts | 15 ++++++++++++--- .../src/api/impl/beacon/blocks/index.ts | 5 +++-- 5 files changed, 32 insertions(+), 15 deletions(-) diff --git a/packages/api/src/beacon/routes/beacon/block.ts b/packages/api/src/beacon/routes/beacon/block.ts index 95a32097028d..c1f1e723e358 100644 --- a/packages/api/src/beacon/routes/beacon/block.ts +++ b/packages/api/src/beacon/routes/beacon/block.ts @@ -229,8 +229,12 @@ export type Api = { * Retrieves BlobSidecar included in requested block. * @param blockId Block identifier. * Can be one of: "head" (canonical head in node's view), "genesis", "finalized", \, \. + * @param indices Array of indices for blob sidecars to request for in the specified block. Returns all blob sidecars in the block if not specified. */ - getBlobSidecars(blockId: BlockId): Promise< + getBlobSidecars( + blockId: BlockId, + indices?: number[] + ): Promise< ApiClientResponse<{ [HttpStatusCode.OK]: {executionOptimistic: ExecutionOptimistic; data: deneb.BlobSidecars}; }> @@ -270,7 +274,7 @@ export type ReqTypes = { publishBlockV2: {body: unknown; query: {broadcast_validation?: string}}; publishBlindedBlock: {body: unknown}; publishBlindedBlockV2: {body: unknown; query: {broadcast_validation?: string}}; - getBlobSidecars: BlockIdOnlyReq; + getBlobSidecars: {params: {block_id: string}; query: {indices?: number[]}}; }; export function getReqSerializers(config: ChainForkConfig): ReqSerializers { @@ -356,7 +360,16 @@ export function getReqSerializers(config: ChainForkConfig): ReqSerializers ({ + params: {block_id: String(block_id)}, + query: {indices}, + }), + parseReq: ({params, query}) => [params.block_id, query.indices], + schema: { + query: {indices: Schema.UintArray}, + }, + }, }; } diff --git a/packages/api/test/unit/beacon/oapiSpec.test.ts b/packages/api/test/unit/beacon/oapiSpec.test.ts index 15a10bfeb6f7..f7d6cb9a077c 100644 --- a/packages/api/test/unit/beacon/oapiSpec.test.ts +++ b/packages/api/test/unit/beacon/oapiSpec.test.ts @@ -130,12 +130,6 @@ const ignoredProperties: Record = { */ getHealth: {request: ["query.syncing_status"]}, - /** - * https://github.com/ChainSafe/lodestar/issues/6185 - * - must have required property 'query' - */ - getBlobSidecars: {request: ["query"]}, - /* https://github.com/ChainSafe/lodestar/issues/4638 /query - must have required property 'skip_randao_verification' diff --git a/packages/api/test/unit/beacon/testData/beacon.ts b/packages/api/test/unit/beacon/testData/beacon.ts index 7fa8368c590b..6d6bc6576f56 100644 --- a/packages/api/test/unit/beacon/testData/beacon.ts +++ b/packages/api/test/unit/beacon/testData/beacon.ts @@ -71,7 +71,7 @@ export const testData: GenericServerTestCases = { res: undefined, }, getBlobSidecars: { - args: ["head"], + args: ["head", [0]], res: {executionOptimistic: true, data: ssz.deneb.BlobSidecars.defaultValue()}, }, diff --git a/packages/api/test/utils/checkAgainstSpec.ts b/packages/api/test/utils/checkAgainstSpec.ts index ed65279bca22..c887f66e95e6 100644 --- a/packages/api/test/utils/checkAgainstSpec.ts +++ b/packages/api/test/utils/checkAgainstSpec.ts @@ -199,12 +199,21 @@ function prettyAjvErrors(errors: ErrorObject[] | null | undefined): string { return errors.map((e) => `${e.instancePath ?? "."} - ${e.message}`).join("\n"); } +type StringifiedProperty = string | StringifiedProperty[]; + +function stringifyProperty(value: unknown): StringifiedProperty { + if (typeof value === "number") { + return value.toString(10); + } else if (Array.isArray(value)) { + return value.map(stringifyProperty); + } + return String(value); +} + function stringifyProperties(obj: Record): Record { for (const key of Object.keys(obj)) { const value = obj[key]; - if (typeof value === "number") { - obj[key] = value.toString(10); - } + obj[key] = stringifyProperty(value); } return obj; diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index f2e29f00fe57..6fde04bc737c 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -406,7 +406,7 @@ export function getBeaconBlockApi({ await publishBlock(signedBlockOrContents, opts); }, - async getBlobSidecars(blockId) { + async getBlobSidecars(blockId, indices) { const {block, executionOptimistic} = await resolveBlockId(chain, blockId); const blockRoot = config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message); @@ -418,9 +418,10 @@ export function getBeaconBlockApi({ if (!blobSidecars) { throw Error(`blobSidecars not found in db for slot=${block.message.slot} root=${toHexString(blockRoot)}`); } + return { executionOptimistic, - data: blobSidecars, + data: indices ? blobSidecars.filter(({index}) => indices.includes(index)) : blobSidecars, }; }, }; From 54c20699d2395ba0ac658616b8a227fc52ccc414 Mon Sep 17 00:00:00 2001 From: Nico Flaig Date: Mon, 29 Jan 2024 18:15:27 +0100 Subject: [PATCH 6/8] fix: ignore stale keystore lockfiles (#6363) * fix: ignore stale keystore lockfiles * Update error message if lockfile is already acquired * Update keymanager lockfile e2e tests --- packages/cli/package.json | 4 +- packages/cli/src/util/lockfile.ts | 54 +++++++++++-------- .../test/e2e/importKeystoresFromApi.test.ts | 8 +-- .../decryptKeystoreDefinitions.test.ts | 13 +++-- packages/db/src/controller/level.ts | 2 +- yarn.lock | 19 +++---- 6 files changed, 55 insertions(+), 45 deletions(-) diff --git a/packages/cli/package.json b/packages/cli/package.json index c16baf13c25f..8531db5b633f 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -77,7 +77,6 @@ "@lodestar/utils": "^1.15.0", "@lodestar/validator": "^1.15.0", "@multiformats/multiaddr": "^12.1.3", - "@types/lockfile": "^1.0.2", "bip39": "^3.1.0", "deepmerge": "^4.3.1", "ethers": "^6.7.0", @@ -86,9 +85,9 @@ "got": "^11.8.6", "inquirer": "^9.1.5", "js-yaml": "^4.1.0", - "lockfile": "^1.0.4", "lodash": "^4.17.21", "prom-client": "^15.1.0", + "proper-lockfile": "^4.1.2", "rimraf": "^4.4.1", "source-map-support": "^0.5.21", "uint8arrays": "^4.0.9", @@ -102,6 +101,7 @@ "@types/got": "^9.6.12", "@types/inquirer": "^9.0.3", "@types/lodash": "^4.14.192", + "@types/proper-lockfile": "^4.1.4", "@types/yargs": "^17.0.24" } } diff --git a/packages/cli/src/util/lockfile.ts b/packages/cli/src/util/lockfile.ts index 65933e8f2897..f7a6ddf4d57d 100644 --- a/packages/cli/src/util/lockfile.ts +++ b/packages/cli/src/util/lockfile.ts @@ -1,29 +1,21 @@ -export type Lockfile = { - lockSync(path: string): void; - unlockSync(path: string): void; -}; - -const lockFile: Lockfile = (await import("lockfile")) as Lockfile; - -function getLockFilepath(filepath: string): string { - return `${filepath}.lock`; -} - -/** - * When lockfile is imported, it registers listeners to process - * Since it's only used by the validator client, require lazily to not pollute - * beacon_node client context - */ -function getLockFile(): Lockfile { - return lockFile; -} +import {lockSync, unlockSync} from "proper-lockfile"; /** * Creates a .lock file for `filepath`, argument passed must not be the lock path * @param filepath File to lock, i.e. `keystore_0001.json` */ export function lockFilepath(filepath: string): void { - getLockFile().lockSync(getLockFilepath(filepath)); + try { + lockSync(filepath, { + // Allows to lock files that do not exist + realpath: false, + }); + } catch (e) { + if (isLockfileError(e) && e.code === "ELOCKED") { + e.message = `${filepath} is already in use by another process`; + } + throw e; + } } /** @@ -31,7 +23,23 @@ export function lockFilepath(filepath: string): void { * @param filepath File to unlock, i.e. `keystore_0001.json` */ export function unlockFilepath(filepath: string): void { - // Does not throw if the lock file is already deleted - // https://github.com/npm/lockfile/blob/6590779867ee9bdc5dbebddc962640759892bb91/lockfile.js#L68 - getLockFile().unlockSync(getLockFilepath(filepath)); + try { + unlockSync(filepath, { + // Allows to unlock files that do not exist + realpath: false, + }); + } catch (e) { + if (isLockfileError(e) && e.code === "ENOTACQUIRED") { + // Do not throw if the lock file is already deleted + return; + } + throw e; + } +} + +// https://github.com/moxystudio/node-proper-lockfile/blob/9f8c303c91998e8404a911dc11c54029812bca69/lib/lockfile.js#L53 +export type LockfileError = Error & {code: "ELOCKED" | "ENOTACQUIRED"}; + +function isLockfileError(e: unknown): e is LockfileError { + return e instanceof Error && (e as LockfileError).code !== undefined; } diff --git a/packages/cli/test/e2e/importKeystoresFromApi.test.ts b/packages/cli/test/e2e/importKeystoresFromApi.test.ts index bb91d467b86a..1cf5f107e226 100644 --- a/packages/cli/test/e2e/importKeystoresFromApi.test.ts +++ b/packages/cli/test/e2e/importKeystoresFromApi.test.ts @@ -95,10 +95,10 @@ describe("import keystores from api", function () { validator.on("exit", (code) => { if (code !== null && code > 0) { // process should exit with code > 0, and an error related to locks. Sample error: - // vc 351591: ✖ Error: EEXIST: file already exists, open '/tmp/tmp-351554-dMctEAj7sJIz/import-keystores-test/keystores/0x8be678633e927aa0435addad5dcd5283fef6110d91362519cd6d43e61f6c017d724fa579cc4b2972134e050b6ba120c0/voting-keystore.json.lock' - // at Object.openSync (node:fs:585:3) - // at Module.exports.lockSync (/home/lion/Code/eth2.0/lodestar/node_modules/lockfile/lockfile.js:277:17) - if (/EEXIST.*voting-keystore\.json\.lock/.test(vcProc2Stderr.read())) { + // vc 351591: ✖ Error: /tmp/tmp-5080-lwNxdM5Ok9ya/import-keystores-test/keystores/0x8be678633e927aa0435addad5dcd5283fef6110d91362519cd6d43e61f6c017d724fa579cc4b2972134e050b6ba120c0/voting-keystore.json is already in use by another process + // at /home/runner/actions-runner/_work/lodestar/lodestar/node_modules/proper-lockfile/lib/lockfile.js:68:47 + // ... more stack trace + if (/Error.*voting-keystore\.json is already in use by another process/.test(vcProc2Stderr.read())) { resolve(); } else { reject(Error(`Second validator proc exited with unknown error. stderr:\n${vcProc2Stderr.read()}`)); diff --git a/packages/cli/test/unit/validator/decryptKeystoreDefinitions.test.ts b/packages/cli/test/unit/validator/decryptKeystoreDefinitions.test.ts index f24b83ae43a6..0f4173604405 100644 --- a/packages/cli/test/unit/validator/decryptKeystoreDefinitions.test.ts +++ b/packages/cli/test/unit/validator/decryptKeystoreDefinitions.test.ts @@ -7,6 +7,7 @@ import {cachedSeckeysHex} from "../../utils/cachedKeys.js"; import {testFilesDir} from "../../utils.js"; import {decryptKeystoreDefinitions} from "../../../src/cmds/validator/keymanager/decryptKeystoreDefinitions.js"; import {LocalKeystoreDefinition} from "../../../src/cmds/validator/keymanager/interface.js"; +import {LockfileError, unlockFilepath} from "../../../src/util/lockfile.js"; describe("decryptKeystoreDefinitions", () => { vi.setConfig({testTimeout: 100_000}); @@ -22,6 +23,10 @@ describe("decryptKeystoreDefinitions", () => { let definitions: LocalKeystoreDefinition[] = []; beforeEach(async () => { + // remove lockfiles from proper-lockfile cache + for (const {keystorePath} of definitions) { + unlockFilepath(keystorePath); + } rimraf.sync(dataDir); rimraf.sync(importFromDir); @@ -46,7 +51,9 @@ describe("decryptKeystoreDefinitions", () => { expect(fs.existsSync(cacheFilePath)).toBe(true); // remove lockfiles created during cache file preparation - rimraf.sync(path.join(importFromDir, "*.lock"), {glob: true}); + for (const {keystorePath} of definitions) { + unlockFilepath(keystorePath); + } }); testDecryptKeystoreDefinitions(cacheFilePath); @@ -75,14 +82,14 @@ describe("decryptKeystoreDefinitions", () => { await decryptKeystoreDefinitions(definitions, {logger: console, signal, cacheFilePath}); expect.fail("Second decrypt should fail due to failure to get lockfile"); } catch (e) { - expect((e as Error).message.startsWith("EEXIST: file already exists")).toBe(true); + expect((e as LockfileError).code).toBe("ELOCKED"); } }); it("decrypt keystores if lockfiles already exist if ignoreLockFile=true", async () => { await decryptKeystoreDefinitions(definitions, {logger: console, signal, cacheFilePath}); - // lockfiles should exist after the first run + await decryptKeystoreDefinitions(definitions, {logger: console, signal, cacheFilePath, ignoreLockFile: true}); }); } diff --git a/packages/db/src/controller/level.ts b/packages/db/src/controller/level.ts index 3eed75958e3e..2cea8681c95b 100644 --- a/packages/db/src/controller/level.ts +++ b/packages/db/src/controller/level.ts @@ -52,7 +52,7 @@ export class LevelDbController implements DatabaseController Date: Tue, 30 Jan 2024 00:17:23 +0700 Subject: [PATCH 7/8] fix: stabilize unknown block sync e2e test (#6364) fix: stabalize unknown block sync e2e test --- .../test/e2e/sync/unknownBlockSync.test.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts index a51beaf7b961..e64adfc94888 100644 --- a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts +++ b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts @@ -12,7 +12,7 @@ import {waitForEvent} from "../../utils/events/resolver.js"; import {getAndInitDevValidators} from "../../utils/node/validator.js"; import {ChainEvent} from "../../../src/chain/index.js"; import {NetworkEvent} from "../../../src/network/index.js"; -import {connect} from "../../utils/network.js"; +import {connect, onPeerConnect} from "../../utils/network.js"; import {testLogger, LogLevel, TestLoggerOpts} from "../../utils/logger.js"; import {BlockError, BlockErrorCode} from "../../../src/chain/errors/index.js"; import {BlockSource, getBlockInput} from "../../../src/chain/blocks/types.js"; @@ -48,7 +48,7 @@ describe("sync / unknown block sync", function () { for (const {id, event} of testCases) { it(id, async function () { // the node needs time to transpile/initialize bls worker threads - const genesisSlotsDelay = 7; + const genesisSlotsDelay = 4; const genesisTime = Math.floor(Date.now() / 1000) + genesisSlotsDelay * testParams.SECONDS_PER_SLOT; const testLoggerOpts: TestLoggerOpts = { level: LogLevel.info, @@ -71,6 +71,7 @@ describe("sync / unknown block sync", function () { chain: {blsVerifyAllMainThread: true}, }, validatorCount, + genesisTime, logger: loggerNodeA, }); @@ -100,7 +101,7 @@ describe("sync / unknown block sync", function () { chain: {blsVerifyAllMainThread: true}, }, validatorCount, - genesisTime: bn.chain.getHeadState().genesisTime, + genesisTime, logger: loggerNodeB, }); @@ -116,7 +117,11 @@ describe("sync / unknown block sync", function () { ({block}) => block === headSummary.blockRoot ); + const connected = Promise.all([onPeerConnect(bn2.network), onPeerConnect(bn.network)]); await connect(bn2.network, bn.network); + await connected; + loggerNodeA.info("Node A connected to Node B"); + const headInput = getBlockInput.preDeneb(config, head, BlockSource.gossip, null); switch (event) { @@ -147,4 +152,4 @@ describe("sync / unknown block sync", function () { await waitForSynced; }); } -}, {timeout: 30_000}); +}, {timeout: 40_000}); From ed43a988bf7acbe25fad9988434a1ade3ed6016e Mon Sep 17 00:00:00 2001 From: Julien Date: Mon, 29 Jan 2024 09:53:35 -0800 Subject: [PATCH 8/8] chore: align vitest config filenames (#6369) --- CONTRIBUTING.md | 2 +- packages/beacon-node/package.json | 10 +++++----- .../{vitest.config.e2e.ts => vitest.e2e.config.ts} | 0 .../{vitest.config.spec.ts => vitest.spec.config.ts} | 0 packages/cli/package.json | 2 +- .../cli/{vitest.config.e2e.ts => vitest.e2e.config.ts} | 0 packages/logger/package.json | 2 +- .../{vitest.config.e2e.ts => vitest.e2e.config.ts} | 0 packages/params/package.json | 2 +- .../{vitest.config.e2e.ts => vitest.e2e.config.ts} | 0 packages/prover/package.json | 2 +- .../{vitest.config.e2e.ts => vitest.e2e.config.ts} | 0 packages/spec-test-util/package.json | 2 +- .../{vitest.config.e2e.ts => vitest.e2e.config.ts} | 0 packages/validator/package.json | 4 ++-- .../{vitest.config.e2e.ts => vitest.e2e.config.ts} | 0 .../{vitest.config.spec.ts => vitest.spec.config.ts} | 0 17 files changed, 13 insertions(+), 13 deletions(-) rename packages/beacon-node/{vitest.config.e2e.ts => vitest.e2e.config.ts} (100%) rename packages/beacon-node/{vitest.config.spec.ts => vitest.spec.config.ts} (100%) rename packages/cli/{vitest.config.e2e.ts => vitest.e2e.config.ts} (100%) rename packages/logger/{vitest.config.e2e.ts => vitest.e2e.config.ts} (100%) rename packages/params/{vitest.config.e2e.ts => vitest.e2e.config.ts} (100%) rename packages/prover/{vitest.config.e2e.ts => vitest.e2e.config.ts} (100%) rename packages/spec-test-util/{vitest.config.e2e.ts => vitest.e2e.config.ts} (100%) rename packages/validator/{vitest.config.e2e.ts => vitest.e2e.config.ts} (100%) rename packages/validator/{vitest.config.spec.ts => vitest.spec.config.ts} (100%) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4074c47e88e8..75fc18a95da9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -65,7 +65,7 @@ If you observe following error running any of the test files that means you are - To then run only that failed test you can run against a specific file as use vitest's filters to run only one case ```sh -LODESTAR_PRESET=minimal yarn vitest --run --config vitest.config.spec.ts test/spec/phase0/sanity.test.ts +LODESTAR_PRESET=minimal yarn vitest --run --config vitest.spec.config.ts test/spec/phase0/sanity.test.ts ``` ## Docker diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index aab75c458d56..174ed8cbb125 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -79,17 +79,17 @@ "test:unit:minimal": "vitest --run --segfaultRetry 3 --dir test/unit/", "test:unit:mainnet": "LODESTAR_PRESET=mainnet vitest --run --dir test/unit-mainnet", "test:unit": "wrapper() { yarn test:unit:minimal $@ && yarn test:unit:mainnet $@; }; wrapper", - "test:e2e": "LODESTAR_PRESET=minimal vitest --run --segfaultRetry 3 --config vitest.config.e2e.ts --dir test/e2e", + "test:e2e": "LODESTAR_PRESET=minimal vitest --run --segfaultRetry 3 --config vitest.e2e.config.ts --dir test/e2e", "test:sim": "vitest --run test/sim/**/*.test.ts", "test:sim:merge-interop": "vitest --run test/sim/merge-interop.test.ts", "test:sim:mergemock": "vitest --run test/sim/mergemock.test.ts", "test:sim:withdrawals": "vitest --run test/sim/withdrawal-interop.test.ts", "test:sim:blobs": "vitest --run test/sim/4844-interop.test.ts", "download-spec-tests": "node --loader=ts-node/esm test/spec/downloadTests.ts", - "test:spec:bls": "vitest --run --config vitest.config.spec.ts --dir test/spec/bls/", - "test:spec:general": "vitest --run --config vitest.config.spec.ts --dir test/spec/general/", - "test:spec:minimal": "LODESTAR_PRESET=minimal vitest --run --config vitest.config.spec.ts --dir test/spec/presets/", - "test:spec:mainnet": "LODESTAR_PRESET=mainnet vitest --run --config vitest.config.spec.ts --dir test/spec/presets/", + "test:spec:bls": "vitest --run --config vitest.spec.config.ts --dir test/spec/bls/", + "test:spec:general": "vitest --run --config vitest.spec.config.ts --dir test/spec/general/", + "test:spec:minimal": "LODESTAR_PRESET=minimal vitest --run --config vitest.spec.config.ts --dir test/spec/presets/", + "test:spec:mainnet": "LODESTAR_PRESET=mainnet vitest --run --config vitest.spec.config.ts --dir test/spec/presets/", "test:spec": "yarn test:spec:bls && yarn test:spec:general && yarn test:spec:minimal && yarn test:spec:mainnet", "check-readme": "typescript-docs-verifier" }, diff --git a/packages/beacon-node/vitest.config.e2e.ts b/packages/beacon-node/vitest.e2e.config.ts similarity index 100% rename from packages/beacon-node/vitest.config.e2e.ts rename to packages/beacon-node/vitest.e2e.config.ts diff --git a/packages/beacon-node/vitest.config.spec.ts b/packages/beacon-node/vitest.spec.config.ts similarity index 100% rename from packages/beacon-node/vitest.config.spec.ts rename to packages/beacon-node/vitest.spec.config.ts diff --git a/packages/cli/package.json b/packages/cli/package.json index 8531db5b633f..d21e523696ef 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -31,7 +31,7 @@ "lint": "eslint --color --ext .ts src/ test/", "lint:fix": "yarn run lint --fix", "test:unit": "vitest --run --dir test/unit/", - "test:e2e": "vitest --run --config vitest.config.e2e.ts --dir test/e2e/", + "test:e2e": "vitest --run --config vitest.e2e.config.ts --dir test/e2e/", "test:sim:multifork": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/multi_fork.test.ts", "test:sim:mixedclient": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/mixed_client.test.ts", "test:sim:endpoints": "LODESTAR_PRESET=minimal node --loader ts-node/esm test/sim/endpoints.test.ts", diff --git a/packages/cli/vitest.config.e2e.ts b/packages/cli/vitest.e2e.config.ts similarity index 100% rename from packages/cli/vitest.config.e2e.ts rename to packages/cli/vitest.e2e.config.ts diff --git a/packages/logger/package.json b/packages/logger/package.json index fd82a7651b3f..e8d32622ae95 100644 --- a/packages/logger/package.json +++ b/packages/logger/package.json @@ -61,7 +61,7 @@ "test:browsers:chrome": "vitest --run --browser chrome --config ./vitest.browser.config.ts --dir test/unit", "test:browsers:firefox": "vitest --run --browser firefox --config ./vitest.browser.config.ts --dir test/unit", "test:browsers:electron": "echo 'Electron tests will be introduced back in the future as soon vitest supports electron.'", - "test:e2e": "LODESTAR_PRESET=minimal vitest --run --config vitest.config.e2e.ts --dir test/e2e", + "test:e2e": "LODESTAR_PRESET=minimal vitest --run --config vitest.e2e.config.ts --dir test/e2e", "check-readme": "typescript-docs-verifier" }, "types": "lib/index.d.ts", diff --git a/packages/logger/vitest.config.e2e.ts b/packages/logger/vitest.e2e.config.ts similarity index 100% rename from packages/logger/vitest.config.e2e.ts rename to packages/logger/vitest.e2e.config.ts diff --git a/packages/params/package.json b/packages/params/package.json index ebd4116f16b1..bd5259c4712d 100644 --- a/packages/params/package.json +++ b/packages/params/package.json @@ -58,7 +58,7 @@ "test:browsers:chrome": "vitest --run --browser chrome --config ./vitest.browser.config.ts --dir test/unit", "test:browsers:firefox": "vitest --run --browser firefox --config ./vitest.browser.config.ts --dir test/unit", "test:browsers:electron": "echo 'Electron tests will be introduced back in the future as soon vitest supports electron.'", - "test:e2e": "LODESTAR_PRESET=minimal vitest --run --config vitest.config.e2e.ts --dir test/e2e/", + "test:e2e": "LODESTAR_PRESET=minimal vitest --run --config vitest.e2e.config.ts --dir test/e2e/", "check-readme": "typescript-docs-verifier" }, "repository": { diff --git a/packages/params/vitest.config.e2e.ts b/packages/params/vitest.e2e.config.ts similarity index 100% rename from packages/params/vitest.config.e2e.ts rename to packages/params/vitest.e2e.config.ts diff --git a/packages/prover/package.json b/packages/prover/package.json index 36433fa8d567..b5c7eac31033 100644 --- a/packages/prover/package.json +++ b/packages/prover/package.json @@ -57,7 +57,7 @@ "test:browsers:chrome": "vitest --run --browser chrome --config ./vitest.browser.config.ts --dir test/unit", "test:browsers:firefox": "vitest --run --browser firefox --config ./vitest.browser.config.ts --dir test/unit", "test:browsers:electron": "echo 'Electron tests will be introduced back in the future as soon vitest supports electron.'", - "test:e2e": "LODESTAR_PRESET=minimal vitest --run --config vitest.config.e2e.ts --dir test/e2e", + "test:e2e": "LODESTAR_PRESET=minimal vitest --run --config vitest.e2e.config.ts --dir test/e2e", "check-readme": "typescript-docs-verifier", "generate-fixtures": "node --loader ts-node/esm scripts/generate_fixtures.ts" }, diff --git a/packages/prover/vitest.config.e2e.ts b/packages/prover/vitest.e2e.config.ts similarity index 100% rename from packages/prover/vitest.config.e2e.ts rename to packages/prover/vitest.e2e.config.ts diff --git a/packages/spec-test-util/package.json b/packages/spec-test-util/package.json index c2c202a72614..7e9a82b16035 100644 --- a/packages/spec-test-util/package.json +++ b/packages/spec-test-util/package.json @@ -48,7 +48,7 @@ "lint:fix": "yarn run lint --fix", "test": "yarn test:unit && yarn test:e2e", "test:unit": "vitest --run --passWithNoTests --dir test/unit/", - "test:e2e": "vitest --run --config vitest.config.e2e.ts --dir test/e2e/", + "test:e2e": "vitest --run --config vitest.e2e.config.ts --dir test/e2e/", "check-readme": "typescript-docs-verifier" }, "repository": { diff --git a/packages/spec-test-util/vitest.config.e2e.ts b/packages/spec-test-util/vitest.e2e.config.ts similarity index 100% rename from packages/spec-test-util/vitest.config.e2e.ts rename to packages/spec-test-util/vitest.e2e.config.ts diff --git a/packages/validator/package.json b/packages/validator/package.json index 5a2dcb55c2ba..5b913ad84a63 100644 --- a/packages/validator/package.json +++ b/packages/validator/package.json @@ -29,8 +29,8 @@ "lint:fix": "yarn run lint --fix", "test:unit": "vitest --run --dir test/unit/", "test": "yarn test:unit && yarn test:e2e", - "test:spec": "vitest --run --config vitest.config.spec.ts --dir test/spec/", - "test:e2e": "LODESTAR_PRESET=mainnet vitest --run --config vitest.config.e2e.ts --dir test/e2e", + "test:spec": "vitest --run --config vitest.spec.config.ts --dir test/spec/", + "test:e2e": "LODESTAR_PRESET=mainnet vitest --run --config vitest.e2e.config.ts --dir test/e2e", "download-spec-tests": "node --loader=ts-node/esm test/spec/downloadTests.ts", "coverage": "codecov -F lodestar-validator", "check-readme": "typescript-docs-verifier" diff --git a/packages/validator/vitest.config.e2e.ts b/packages/validator/vitest.e2e.config.ts similarity index 100% rename from packages/validator/vitest.config.e2e.ts rename to packages/validator/vitest.e2e.config.ts diff --git a/packages/validator/vitest.config.spec.ts b/packages/validator/vitest.spec.config.ts similarity index 100% rename from packages/validator/vitest.config.spec.ts rename to packages/validator/vitest.spec.config.ts