Merge branch 'unstable' into nflaig/block-production-selection

ChainSafe · Oct 29, 2024 · 28d4487 · 28d4487
2 parents 13faa80 + 3cd2f16
commit 28d4487
Show file tree

Hide file tree

Showing 50 changed files with 715 additions and 769 deletions.
diff --git a/dashboards/lodestar_validator_client.json b/dashboards/lodestar_validator_client.json
@@ -497,7 +497,7 @@
       },
       "gridPos": {
         "h": 3,
-        "w": 6,
+        "w": 3,
         "x": 12,
         "y": 2
       },
@@ -552,8 +552,8 @@
       },
       "gridPos": {
         "h": 3,
-        "w": 6,
-        "x": 18,
+        "w": 3,
+        "x": 15,
         "y": 2
       },
       "id": 37,
@@ -590,6 +590,91 @@
       "title": "Heap used",
       "type": "stat"
     },
+    {
+      "datasource": {
+        "type": "prometheus",
+        "uid": "${DS_PROMETHEUS}"
+      },
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "thresholds"
+          },
+          "custom": {
+            "align": "center",
+            "cellOptions": {
+              "type": "color-text"
+            },
+            "inspect": false
+          },
+          "mappings": []
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 3,
+        "w": 6,
+        "x": 18,
+        "y": 2
+      },
+      "id": 48,
+      "options": {
+        "cellHeight": "sm",
+        "footer": {
+          "countRows": false,
+          "enablePagination": false,
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true
+      },
+      "pluginVersion": "10.4.1",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "editorMode": "code",
+          "exemplar": false,
+          "expr": "vc_default_configuration",
+          "format": "table",
+          "instant": true,
+          "legendFormat": "__auto",
+          "range": false,
+          "refId": "A"
+        }
+      ],
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "Value": true,
+              "__name__": true,
+              "client_name": true,
+              "group": true,
+              "host_type": true,
+              "instance": true,
+              "job": true,
+              "network": true,
+              "scrape_location": true
+            },
+            "includeByName": {},
+            "indexByName": {},
+            "renameByName": {
+              "broadcastValidation": "Broadcast validation",
+              "builderSelection": "Builder selection"
+            }
+          }
+        }
+      ],
+      "type": "table"
+    },
     {
       "datasource": {
         "type": "prometheus",

diff --git a/package.json b/package.json
@@ -60,7 +60,7 @@
     "https-browserify": "^1.0.0",
     "jsdom": "^23.0.1",
     "lerna": "^7.3.0",
-    "libp2p": "2.1.7",
+    "libp2p": "1.4.3",
     "mocha": "^10.2.0",
     "node-gyp": "^9.4.0",
     "npm-run-all": "^4.1.5",

diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json
@@ -96,10 +96,11 @@
   "dependencies": {
     "@chainsafe/as-sha256": "^0.5.0",
     "@chainsafe/blst": "^2.1.0",
-    "@chainsafe/discv5": "^10.0.1",
-    "@chainsafe/enr": "^4.0.1",
-    "@chainsafe/libp2p-gossipsub": "^14.1.0",
-    "@chainsafe/libp2p-noise": "^16.0.0",
+    "@chainsafe/discv5": "^9.0.0",
+    "@chainsafe/enr": "^3.0.0",
+    "@chainsafe/libp2p-gossipsub": "^13.0.0",
+    "@chainsafe/libp2p-identify": "^1.0.0",
+    "@chainsafe/libp2p-noise": "^15.0.0",
     "@chainsafe/persistent-merkle-tree": "^0.8.0",
     "@chainsafe/prometheus-gc-stats": "^1.0.0",
     "@chainsafe/ssz": "^0.18.0",
@@ -110,15 +111,15 @@
     "@fastify/cors": "^10.0.1",
     "@fastify/swagger": "^9.0.0",
     "@fastify/swagger-ui": "^5.0.1",
-    "@libp2p/bootstrap": "^11.0.4",
-    "@libp2p/crypto": "^5.0.4",
-    "@libp2p/identify": "^3.0.4",
-    "@libp2p/interface": "^2.1.2",
-    "@libp2p/mdns": "^11.0.4",
-    "@libp2p/mplex": "^11.0.4",
-    "@libp2p/peer-id": "^5.0.4",
-    "@libp2p/prometheus-metrics": "^4.1.2",
-    "@libp2p/tcp": "10.0.4",
+    "@libp2p/bootstrap": "^10.0.21",
+    "@libp2p/identify": "^1.0.20",
+    "@libp2p/interface": "^1.3.0",
+    "@libp2p/mdns": "^10.0.21",
+    "@libp2p/mplex": "^10.0.21",
+    "@libp2p/peer-id": "^4.1.0",
+    "@libp2p/peer-id-factory": "^4.1.0",
+    "@libp2p/prometheus-metrics": "^3.0.21",
+    "@libp2p/tcp": "9.0.23",
     "@lodestar/api": "^1.22.0",
     "@lodestar/config": "^1.22.0",
     "@lodestar/db": "^1.22.0",
@@ -133,15 +134,15 @@
     "@lodestar/validator": "^1.22.0",
     "@multiformats/multiaddr": "^12.1.3",
     "c-kzg": "^2.1.2",
-    "datastore-core": "^10.0.0",
-    "datastore-level": "^11.0.0",
+    "datastore-core": "^9.1.1",
+    "datastore-level": "^10.1.1",
     "deepmerge": "^4.3.1",
     "fastify": "^5.0.0",
-    "interface-datastore": "^8.3.0",
+    "interface-datastore": "^8.2.7",
     "it-all": "^3.0.4",
     "it-pipe": "^3.0.1",
     "jwt-simple": "0.5.6",
-    "libp2p": "2.1.7",
+    "libp2p": "1.4.3",
     "multiformats": "^11.0.1",
     "prom-client": "^15.1.0",
     "qs": "^6.11.1",

diff --git a/packages/beacon-node/src/network/core/networkCore.ts b/packages/beacon-node/src/network/core/networkCore.ts
@@ -1,4 +1,4 @@
-import {Connection, PrivateKey} from "@libp2p/interface";
+import {Connection, PeerId} from "@libp2p/interface";
 import {multiaddr} from "@multiformats/multiaddr";
 import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types";
 import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/dist/src/score/peer-score.js";
@@ -55,7 +55,7 @@ type Mods = {
 export type BaseNetworkInit = {
   opts: NetworkOptions;
   config: BeaconConfig;
-  privateKey: PrivateKey;
+  peerId: PeerId;
   peerStoreDir: string | undefined;
   logger: LoggerNode;
   metricsRegistry: RegistryMetricCreator | null;
@@ -126,7 +126,7 @@ export class NetworkCore implements INetworkCore {
   static async init({
     opts,
     config,
-    privateKey,
+    peerId,
     peerStoreDir,
     logger,
     metricsRegistry,
@@ -136,7 +136,7 @@ export class NetworkCore implements INetworkCore {
     activeValidatorCount,
     initialStatus,
   }: BaseNetworkInit): Promise<NetworkCore> {
-    const libp2p = await createNodeJsLibp2p(privateKey, opts, {
+    const libp2p = await createNodeJsLibp2p(peerId, opts, {
       peerStoreDir,
       metrics: Boolean(metricsRegistry),
       metricsRegistry: metricsRegistry ?? undefined,
@@ -200,9 +200,8 @@ export class NetworkCore implements INetworkCore {
 
     const peerManager = await PeerManager.init(
       {
-        privateKey,
         libp2p,
-        gossip,
+        gossip: gossip,
         reqResp,
         attnetsService,
         syncnetsService,
@@ -363,11 +362,7 @@ export class NetworkCore implements INetworkCore {
   }
 
   getConnectionsByPeer(): Map<string, Connection[]> {
-    const m = new Map<string, Connection[]>();
-    for (const [k, v] of getConnectionsMap(this.libp2p).entries()) {
-      m.set(k, v.value);
-    }
-    return m;
+    return getConnectionsMap(this.libp2p);
   }
 
   async getConnectedPeers(): Promise<PeerIdStr[]> {

diff --git a/packages/beacon-node/src/network/core/networkCoreWorker.ts b/packages/beacon-node/src/network/core/networkCoreWorker.ts
@@ -3,8 +3,7 @@ import path from "node:path";
 import worker from "node:worker_threads";
 import type {ModuleThread} from "@chainsafe/threads";
 import {expose} from "@chainsafe/threads/worker";
-import {privateKeyFromProtobuf} from "@libp2p/crypto/keys";
-import {peerIdFromPrivateKey} from "@libp2p/peer-id";
+import {createFromProtobuf} from "@libp2p/peer-id-factory";
 import {chainConfigFromJson, createBeaconConfig} from "@lodestar/config";
 import {getNodeLogger} from "@lodestar/logger/node";
 import {RegistryMetricCreator, collectNodeJSMetrics} from "../../metrics/index.js";
@@ -33,8 +32,7 @@ if (!workerData) throw Error("workerData must be defined");
 if (!parentPort) throw Error("parentPort must be defined");
 
 const config = createBeaconConfig(chainConfigFromJson(workerData.chainConfigJson), workerData.genesisValidatorsRoot);
-const privateKey = privateKeyFromProtobuf(workerData.privateKeyProto);
-const peerId = peerIdFromPrivateKey(privateKey);
+const peerId = await createFromProtobuf(workerData.peerIdProto);
 
 // TODO: Pass options from main thread for logging
 // TODO: Logging won't be visible in file loggers
@@ -94,7 +92,7 @@ if (networkCoreWorkerMetrics) {
 const core = await NetworkCore.init({
   opts: workerData.opts,
   config,
-  privateKey,
+  peerId,
   peerStoreDir: workerData.peerStoreDir,
   logger,
   metricsRegistry: metricsRegister,

diff --git a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts
@@ -3,8 +3,8 @@ import workerThreads from "node:worker_threads";
 import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/dist/src/score/peer-score.js";
 import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types";
 import {ModuleThread, Thread, Worker, spawn} from "@chainsafe/threads";
-import {PrivateKey} from "@libp2p/interface";
-import {privateKeyToProtobuf} from "@libp2p/crypto/keys";
+import {PeerId, Secp256k1PeerId} from "@libp2p/interface";
+import {exportToProtobuf} from "@libp2p/peer-id-factory";
 import {routes} from "@lodestar/api";
 import {BeaconConfig, chainConfigToJson} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
@@ -44,7 +44,7 @@ export type WorkerNetworkCoreInitModules = {
   opts: WorkerNetworkCoreOpts;
   config: BeaconConfig;
   logger: LoggerNode;
-  privateKey: PrivateKey;
+  peerId: PeerId;
   events: NetworkEventBus;
   metrics: Metrics | null;
   getReqRespHandler: GetReqRespHandlerFn;
@@ -103,14 +103,14 @@ export class WorkerNetworkCore implements INetworkCore {
   }
 
   static async init(modules: WorkerNetworkCoreInitModules): Promise<WorkerNetworkCore> {
-    const {opts, config, privateKey} = modules;
+    const {opts, config, peerId} = modules;
     const {genesisTime, peerStoreDir, activeValidatorCount, localMultiaddrs, metricsEnabled, initialStatus} = opts;
 
     const workerData: NetworkWorkerData = {
       opts,
       chainConfigJson: chainConfigToJson(config),
       genesisValidatorsRoot: config.genesisValidatorsRoot,
-      privateKeyProto: privateKeyToProtobuf(privateKey),
+      peerIdProto: exportToProtobuf(peerId as Secp256k1PeerId),
       localMultiaddrs,
       metricsEnabled,
       peerStoreDir,

diff --git a/packages/beacon-node/src/network/core/types.ts b/packages/beacon-node/src/network/core/types.ts
@@ -78,7 +78,7 @@ export type NetworkWorkerData = {
   genesisTime: number;
   activeValidatorCount: number;
   initialStatus: phase0.Status;
-  privateKeyProto: Uint8Array;
+  peerIdProto: Uint8Array;
   localMultiaddrs: string[];
   metricsEnabled: boolean;
   peerStoreDir?: string;

diff --git a/packages/beacon-node/src/network/discv5/index.ts b/packages/beacon-node/src/network/discv5/index.ts
@@ -1,16 +1,16 @@
 import EventEmitter from "node:events";
-import {privateKeyToProtobuf} from "@libp2p/crypto/keys";
-import {PrivateKey} from "@libp2p/interface";
+import {PeerId, Secp256k1PeerId} from "@libp2p/interface";
 import {StrictEventEmitter} from "strict-event-emitter-types";
-import {ENR, ENRData, SignableENR} from "@chainsafe/enr";
+import {exportToProtobuf} from "@libp2p/peer-id-factory";
+import {createPrivateKeyFromPeerId, ENR, ENRData, SignableENR} from "@chainsafe/enr";
 import {spawn, Thread, Worker} from "@chainsafe/threads";
 import {chainConfigFromJson, chainConfigToJson, BeaconConfig} from "@lodestar/config";
 import {LoggerNode} from "@lodestar/logger/node";
 import {NetworkCoreMetrics} from "../core/metrics.js";
 import {Discv5WorkerApi, Discv5WorkerData, LodestarDiscv5Opts} from "./types.js";
 
 export type Discv5Opts = {
-  privateKey: PrivateKey;
+  peerId: PeerId;
   discv5: LodestarDiscv5Opts;
   logger: LoggerNode;
   config: BeaconConfig;
@@ -25,6 +25,7 @@ export type Discv5Events = {
  * Wrapper class abstracting the details of discv5 worker instantiation and message-passing
  */
 export class Discv5Worker extends (EventEmitter as {new (): StrictEventEmitter<EventEmitter, Discv5Events>}) {
+  private readonly keypair;
   private readonly subscription: {unsubscribe: () => void};
   private closed = false;
 
@@ -34,13 +35,14 @@ export class Discv5Worker extends (EventEmitter as {new (): StrictEventEmitter<E
   ) {
     super();
 
+    this.keypair = createPrivateKeyFromPeerId(this.opts.peerId);
     this.subscription = workerApi.discovered().subscribe((enrObj) => this.onDiscovered(enrObj));
   }
 
   static async init(opts: Discv5Opts): Promise<Discv5Worker> {
     const workerData: Discv5WorkerData = {
       enr: opts.discv5.enr,
-      privateKeyProto: privateKeyToProtobuf(opts.privateKey),
+      peerIdProto: exportToProtobuf(opts.peerId as Secp256k1PeerId),
       bindAddrs: opts.discv5.bindAddrs,
       config: opts.discv5.config ?? {},
       bootEnrs: opts.discv5.bootEnrs,
@@ -78,7 +80,7 @@ export class Discv5Worker extends (EventEmitter as {new (): StrictEventEmitter<E
 
   async enr(): Promise<SignableENR> {
     const obj = await this.workerApi.enr();
-    return new SignableENR(obj.kvs, obj.seq, this.opts.privateKey.raw);
+    return new SignableENR(obj.kvs, obj.seq, this.keypair.privateKey);
   }
 
   setEnrValue(key: string, value: Uint8Array): Promise<void> {

diff --git a/packages/beacon-node/src/network/discv5/types.ts b/packages/beacon-node/src/network/discv5/types.ts
@@ -31,7 +31,7 @@ export type LodestarDiscv5Opts = {
 /** discv5 worker constructor data */
 export interface Discv5WorkerData {
   enr: string;
-  privateKeyProto: Uint8Array;
+  peerIdProto: Uint8Array;
   bindAddrs: BindAddrs;
   config: Discv5Config;
   bootEnrs: string[];