From 3d1a1e62adefaab774b66d810aa3a5e800f7ab77 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 02:29:23 -0400 Subject: [PATCH 01/42] feat(beacon-node): pass metrics to workerEvents --- packages/beacon-node/src/network/core/metrics.ts | 2 ++ .../src/network/core/networkCoreWorker.ts | 13 ++++++++----- .../src/network/core/networkCoreWorkerHandler.ts | 2 ++ packages/beacon-node/src/util/workerEvents.ts | 4 ++++ 4 files changed, 16 insertions(+), 5 deletions(-) diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts index 78bc88d52fe7..357a89572e84 100644 --- a/packages/beacon-node/src/network/core/metrics.ts +++ b/packages/beacon-node/src/network/core/metrics.ts @@ -333,6 +333,8 @@ export function createNetworkCoreMetrics(register: RegistryMetricCreator) { }; } +export type NetworkCoreWorkerMetrics = ReturnType; + // eslint-disable-next-line @typescript-eslint/explicit-function-return-type export function getNetworkCoreWorkerMetrics(register: RegistryMetricCreator) { return { diff --git a/packages/beacon-node/src/network/core/networkCoreWorker.ts b/packages/beacon-node/src/network/core/networkCoreWorker.ts index a0c8ff22fe60..3fdddc175082 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorker.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorker.ts @@ -13,6 +13,9 @@ import {peerIdToString} from "../../util/peerId.js"; import {profileNodeJS} from "../../util/profile.js"; import {NetworkEventBus, NetworkEventData, networkEventDirection} from "../events.js"; import {wireEventsOnWorkerThread} from "../../util/workerEvents.js"; +import {NetworkCoreWorkerMetrics, getNetworkCoreWorkerMetrics} from "./metrics.js"; +import {NetworkWorkerApi, NetworkWorkerData} from "./types.js"; +import {NetworkCore} from "./networkCore.js"; import { NetworkWorkerThreadEventType, ReqRespBridgeEventBus, @@ -21,9 +24,6 @@ import { getReqRespBridgeRespEvents, reqRespBridgeEventDirection, } from "./events.js"; -import {getNetworkCoreWorkerMetrics} from "./metrics.js"; -import {NetworkCore} from "./networkCore.js"; -import {NetworkWorkerApi, NetworkWorkerData} from "./types.js"; // Cloned data from instantiation const workerData = worker.workerData as NetworkWorkerData; @@ -83,11 +83,12 @@ new AsyncIterableBridgeHandler(getReqRespBridgeReqEvents(reqRespBridgeEventBus), ); const reqRespBridgeRespCaller = new AsyncIterableBridgeCaller(getReqRespBridgeRespEvents(reqRespBridgeEventBus)); +let networkCoreWorkerMetrics: NetworkCoreWorkerMetrics | null = null; // respBridgeCaller metrics if (metricsRegister) { - const networkCoreWorkerMetrics = getNetworkCoreWorkerMetrics(metricsRegister); + networkCoreWorkerMetrics = getNetworkCoreWorkerMetrics(metricsRegister); networkCoreWorkerMetrics.reqRespBridgeRespCallerPending.addCollect(() => { - networkCoreWorkerMetrics.reqRespBridgeRespCallerPending.set(reqRespBridgeRespCaller.pendingCount); + networkCoreWorkerMetrics?.reqRespBridgeRespCallerPending.set(reqRespBridgeRespCaller.pendingCount); }); } @@ -110,12 +111,14 @@ wireEventsOnWorkerThread( NetworkWorkerThreadEventType.networkEvent, events, parentPort, + networkCoreWorkerMetrics, networkEventDirection ); wireEventsOnWorkerThread( NetworkWorkerThreadEventType.reqRespBridgeEvents, reqRespBridgeEventBus, parentPort, + networkCoreWorkerMetrics, reqRespBridgeEventDirection ); diff --git a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts index 73ca9e9c5fd0..8c944dd87d2d 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts @@ -75,12 +75,14 @@ export class WorkerNetworkCore implements INetworkCore { NetworkWorkerThreadEventType.networkEvent, modules.events, modules.worker as unknown as worker_threads.Worker, + modules.metrics, networkEventDirection ); wireEventsOnMainThread( NetworkWorkerThreadEventType.reqRespBridgeEvents, this.reqRespBridgeEventBus, modules.worker as unknown as worker_threads.Worker, + modules.metrics, reqRespBridgeEventDirection ); diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index cd61e6b95393..02d2ff47fba2 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -2,6 +2,8 @@ import {MessagePort, Worker} from "node:worker_threads"; import {Thread} from "@chainsafe/threads"; import {Logger} from "@lodestar/logger"; import {sleep} from "@lodestar/utils"; +import {Metrics} from "../metrics/metrics.js"; +import {NetworkCoreWorkerMetrics} from "../network/core/metrics.js"; import {StrictEventEmitterSingleArg} from "./strictEvents.js"; export type WorkerBridgeEvent = { @@ -27,6 +29,7 @@ export function wireEventsOnWorkerThread( mainEventName: string, events: StrictEventEmitterSingleArg, parentPort: MessagePort, + metrics: NetworkCoreWorkerMetrics | null, isWorkerToMain: {[K in keyof EventData]: EventDirection} ): void { // Subscribe to events from main thread @@ -60,6 +63,7 @@ export function wireEventsOnMainThread( mainEventName: string, events: StrictEventEmitterSingleArg, worker: Pick, + metrics: Metrics | null, isWorkerToMain: {[K in keyof EventData]: EventDirection} ): void { // Subscribe to events from main thread From bd0c2ffb582e56847b51924919c37a54ee08f818 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 05:37:53 -0400 Subject: [PATCH 02/42] feat(beacon-node): add types for network worker event metrics --- packages/beacon-node/src/network/events.ts | 15 ++++++++------- .../beacon-node/src/util/asyncIterableToEvents.ts | 9 +++++---- packages/beacon-node/src/util/types.ts | 2 ++ 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/packages/beacon-node/src/network/events.ts b/packages/beacon-node/src/network/events.ts index 815efda7d9b8..5520287e8eac 100644 --- a/packages/beacon-node/src/network/events.ts +++ b/packages/beacon-node/src/network/events.ts @@ -6,6 +6,7 @@ import {BlockInput} from "../chain/blocks/types.js"; import {StrictEventEmitterSingleArg} from "../util/strictEvents.js"; import {PeerIdStr} from "../util/peerId.js"; import {EventDirection} from "../util/workerEvents.js"; +import {EmittedAt} from "../util/types.js"; import {RequestTypedContainer} from "./reqresp/ReqRespBeaconNode.js"; import {PendingGossipsubMessage} from "./processor/types.js"; @@ -27,13 +28,13 @@ export enum NetworkEvent { } export type NetworkEventData = { - [NetworkEvent.peerConnected]: {peer: PeerIdStr; status: phase0.Status}; - [NetworkEvent.peerDisconnected]: {peer: PeerIdStr}; - [NetworkEvent.reqRespRequest]: {request: RequestTypedContainer; peer: PeerId}; - [NetworkEvent.unknownBlockParent]: {blockInput: BlockInput; peer: PeerIdStr}; - [NetworkEvent.unknownBlock]: {rootHex: RootHex; peer?: PeerIdStr}; - [NetworkEvent.pendingGossipsubMessage]: PendingGossipsubMessage; - [NetworkEvent.gossipMessageValidationResult]: { + [NetworkEvent.peerConnected]: EmittedAt & {peer: PeerIdStr; status: phase0.Status}; + [NetworkEvent.peerDisconnected]: EmittedAt & {peer: PeerIdStr}; + [NetworkEvent.reqRespRequest]: EmittedAt & {request: RequestTypedContainer; peer: PeerId}; + [NetworkEvent.unknownBlockParent]: EmittedAt & {blockInput: BlockInput; peer: PeerIdStr}; + [NetworkEvent.unknownBlock]: EmittedAt & {rootHex: RootHex; peer?: PeerIdStr}; + [NetworkEvent.pendingGossipsubMessage]: EmittedAt & PendingGossipsubMessage; + [NetworkEvent.gossipMessageValidationResult]: EmittedAt & { msgId: string; propagationSource: PeerIdStr; acceptance: TopicValidatorResult; diff --git a/packages/beacon-node/src/util/asyncIterableToEvents.ts b/packages/beacon-node/src/util/asyncIterableToEvents.ts index 0468ca2b431b..339490d550df 100644 --- a/packages/beacon-node/src/util/asyncIterableToEvents.ts +++ b/packages/beacon-node/src/util/asyncIterableToEvents.ts @@ -1,7 +1,8 @@ import {LinkedList} from "./array.js"; import {ThreadBoundaryError, fromThreadBoundaryError, toThreadBoundaryError} from "./error.js"; +import {EmittedAt} from "./types.js"; -export type RequestEvent = { +export type RequestEvent = EmittedAt & { callArgs: T; id: number; }; @@ -13,9 +14,9 @@ export enum IteratorEventType { } export type IteratorEvent = - | {type: IteratorEventType.next; id: number; item: V} - | {type: IteratorEventType.done; id: number} - | {type: IteratorEventType.error; id: number; error: ThreadBoundaryError}; + | (EmittedAt & {type: IteratorEventType.next; id: number; item: V}) + | (EmittedAt & {type: IteratorEventType.done; id: number}) + | (EmittedAt & {type: IteratorEventType.error; id: number; error: ThreadBoundaryError}); export type AsyncIterableEventBus = { emitRequest(data: RequestEvent): void; diff --git a/packages/beacon-node/src/util/types.ts b/packages/beacon-node/src/util/types.ts index 545a706c7511..815d49e20835 100644 --- a/packages/beacon-node/src/util/types.ts +++ b/packages/beacon-node/src/util/types.ts @@ -12,3 +12,5 @@ export const signedBLSToExecutionChangeVersionedType = new ContainerType( {jsonCase: "eth2", typeName: "SignedBLSToExecutionChangeVersionedType"} ); export type SignedBLSToExecutionChangeVersioned = ValueOf; + +export type EmittedAt = {emittedAt: number}; From 1950c788751aca3ee548a0c55687f4ca71698140 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 05:38:24 -0400 Subject: [PATCH 03/42] feat(beacon-node): add metrics for network worker events --- packages/beacon-node/src/metrics/metrics/lodestar.ts | 10 ++++++++++ packages/beacon-node/src/network/core/metrics.ts | 12 +++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 8b8ce0f0c2bc..5aa067b636ec 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -121,6 +121,16 @@ export function createLodestarMetrics( help: "Current count of pending items in reqRespBridgeReqCaller data structure", }), }, + networkWorkerWireEventsOnMainThreadEventLatency: register.histogram({ + name: "lodestar_network_worker_wire_events_on_main_thread_event_latency", + help: "Latency to re-emit network events on main thread event bus", + buckets: [0.001, 0.003, 0.01, 0.03, 0.1], + }), + networkWorkerWireEventsOnMainThreadPortLatency: register.histogram({ + name: "lodestar_network_worker_wire_events_on_main_thread_port_latency", + help: "Latency to transmit network events to main thread across worker port", + buckets: [0.001, 0.003, 0.01, 0.03, 0.1], + }), regenQueue: { length: register.gauge({ diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts index 357a89572e84..b4e66e74daf3 100644 --- a/packages/beacon-node/src/network/core/metrics.ts +++ b/packages/beacon-node/src/network/core/metrics.ts @@ -339,8 +339,18 @@ export type NetworkCoreWorkerMetrics = ReturnType Date: Tue, 25 Jul 2023 05:39:49 -0400 Subject: [PATCH 04/42] feat(beacon-node): add metric data to network worker events --- .../beacon-node/src/network/core/events.ts | 8 ++-- .../src/network/gossip/gossipsub.ts | 1 + .../src/network/peers/peerManager.ts | 2 +- .../src/network/processor/gossipHandlers.ts | 2 +- .../src/network/processor/index.ts | 2 +- .../src/network/reqresp/ReqRespBeaconNode.ts | 2 +- packages/beacon-node/src/util/workerEvents.ts | 38 ++++++++++++------- .../e2e/network/peers/peerManager.test.ts | 3 ++ .../test/e2e/sync/unknownBlockSync.test.ts | 2 + .../test/unit/sync/unknownBlock.test.ts | 3 +- 10 files changed, 41 insertions(+), 22 deletions(-) diff --git a/packages/beacon-node/src/network/core/events.ts b/packages/beacon-node/src/network/core/events.ts index 31bb0cb5d891..ec9cd2711335 100644 --- a/packages/beacon-node/src/network/core/events.ts +++ b/packages/beacon-node/src/network/core/events.ts @@ -35,8 +35,8 @@ export function getReqRespBridgeReqEvents( events: IReqRespBridgeEventBus ): AsyncIterableEventBus { return { - emitRequest: (data) => events.emit(ReqRespBridgeEvent.outgoingRequest, data), - emitResponse: (data) => events.emit(ReqRespBridgeEvent.incomingResponse, data), + emitRequest: (data) => events.emit(ReqRespBridgeEvent.outgoingRequest, {...data, emittedAt: Date.now()}), + emitResponse: (data) => events.emit(ReqRespBridgeEvent.incomingResponse, {...data, emittedAt: Date.now()}), onRequest: (cb) => events.on(ReqRespBridgeEvent.outgoingRequest, cb), onResponse: (cb) => events.on(ReqRespBridgeEvent.incomingResponse, cb), }; @@ -46,8 +46,8 @@ export function getReqRespBridgeRespEvents( events: IReqRespBridgeEventBus ): AsyncIterableEventBus { return { - emitRequest: (data) => events.emit(ReqRespBridgeEvent.incomingRequest, data), - emitResponse: (data) => events.emit(ReqRespBridgeEvent.outgoingResponse, data), + emitRequest: (data) => events.emit(ReqRespBridgeEvent.incomingRequest, {...data, emittedAt: Date.now()}), + emitResponse: (data) => events.emit(ReqRespBridgeEvent.outgoingResponse, {...data, emittedAt: Date.now()}), onRequest: (cb) => events.on(ReqRespBridgeEvent.incomingRequest, cb), onResponse: (cb) => events.on(ReqRespBridgeEvent.outgoingResponse, cb), }; diff --git a/packages/beacon-node/src/network/gossip/gossipsub.ts b/packages/beacon-node/src/network/gossip/gossipsub.ts index 649d8a0f9361..65e9b01bc2d1 100644 --- a/packages/beacon-node/src/network/gossip/gossipsub.ts +++ b/packages/beacon-node/src/network/gossip/gossipsub.ts @@ -283,6 +283,7 @@ export class Eth2Gossipsub extends GossipSub { // See https://github.com/ChainSafe/lodestar/issues/5604 setTimeout(() => { this.events.emit(NetworkEvent.pendingGossipsubMessage, { + emittedAt: Date.now(), topic, msg, msgId, diff --git a/packages/beacon-node/src/network/peers/peerManager.ts b/packages/beacon-node/src/network/peers/peerManager.ts index 0ce3fdbb107c..5b6df685066c 100644 --- a/packages/beacon-node/src/network/peers/peerManager.ts +++ b/packages/beacon-node/src/network/peers/peerManager.ts @@ -377,7 +377,7 @@ export class PeerManager { peerData.relevantStatus = RelevantPeerStatus.relevant; } if (getConnection(this.libp2p, peer.toString())) { - this.networkEventBus.emit(NetworkEvent.peerConnected, {peer: peer.toString(), status}); + this.networkEventBus.emit(NetworkEvent.peerConnected, {emittedAt: Date.now(), peer: peer.toString(), status}); } } diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index 4916b98abfee..a09eefa597fa 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -150,7 +150,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler // Don't trigger this yet if full block and blobs haven't arrived yet if (e instanceof BlockGossipError && e.type.code === BlockErrorCode.PARENT_UNKNOWN && blockInput !== null) { logger.debug("Gossip block has error", {slot, root: blockHex, code: e.type.code}); - events.emit(NetworkEvent.unknownBlockParent, {blockInput, peer: peerIdStr}); + events.emit(NetworkEvent.unknownBlockParent, {emittedAt: Date.now(), blockInput, peer: peerIdStr}); } } diff --git a/packages/beacon-node/src/network/processor/index.ts b/packages/beacon-node/src/network/processor/index.ts index 4abf4baa7472..ce13de72a220 100644 --- a/packages/beacon-node/src/network/processor/index.ts +++ b/packages/beacon-node/src/network/processor/index.ts @@ -234,7 +234,7 @@ export class NetworkProcessor { // Search for the unknown block if (!this.unknownRootsBySlot.getOrDefault(slot).has(root)) { this.unknownRootsBySlot.getOrDefault(slot).add(root); - this.events.emit(NetworkEvent.unknownBlock, {rootHex: root, peer}); + this.events.emit(NetworkEvent.unknownBlock, {emittedAt: Date.now(), rootHex: root, peer}); } } diff --git a/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts b/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts index 69b83ee327c6..19a8678c0c1f 100644 --- a/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts +++ b/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts @@ -259,7 +259,7 @@ export class ReqRespBeaconNode extends ReqResp { // Allow onRequest to return and close the stream // For Goodbye there may be a race condition where the listener of `receivedGoodbye` // disconnects in the same syncronous call, preventing the stream from ending cleanly - setTimeout(() => this.networkEventBus.emit(NetworkEvent.reqRespRequest, {request, peer}), 0); + setTimeout(() => this.networkEventBus.emit(NetworkEvent.reqRespRequest, {emittedAt: Date.now(), request, peer}), 0); } protected onIncomingRequest(peerId: PeerId, protocol: ProtocolDescriptor): void { diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 02d2ff47fba2..8bc1f79852ff 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -5,11 +5,15 @@ import {sleep} from "@lodestar/utils"; import {Metrics} from "../metrics/metrics.js"; import {NetworkCoreWorkerMetrics} from "../network/core/metrics.js"; import {StrictEventEmitterSingleArg} from "./strictEvents.js"; +import {EmittedAt} from "./types.js"; -export type WorkerBridgeEvent = { +type EventData = {[key: string]: EmittedAt}; + +export type WorkerBridgeEvent = { type: string; - event: keyof EventData; - data: EventData[keyof EventData]; + event: keyof E; + posted: number; + data: E[keyof E]; }; export enum EventDirection { @@ -25,22 +29,25 @@ export enum EventDirection { * - worker to main * - main to worker */ -export function wireEventsOnWorkerThread( +export function wireEventsOnWorkerThread( mainEventName: string, - events: StrictEventEmitterSingleArg, + events: StrictEventEmitterSingleArg, parentPort: MessagePort, metrics: NetworkCoreWorkerMetrics | null, - isWorkerToMain: {[K in keyof EventData]: EventDirection} + isWorkerToMain: {[K in keyof E]: EventDirection} ): void { // Subscribe to events from main thread - parentPort.on("message", (data: WorkerBridgeEvent) => { + parentPort.on("message", (data: WorkerBridgeEvent) => { if ( typeof data === "object" && data.type === mainEventName && // This check is not necessary but added for safety in case of improper implemented events isWorkerToMain[data.event] === EventDirection.mainToWorker ) { + const emitted = Date.now(); events.emit(data.event, data.data); + metrics?.networkWorkerWireEventsOnWorkerThreadPortLatency.observe(emitted - data.posted); + metrics?.networkWorkerWireEventsOnWorkerThreadEventLatency.observe(emitted - data.data.emittedAt); } }); @@ -51,6 +58,7 @@ export function wireEventsOnWorkerThread( const workerEvent: WorkerBridgeEvent = { type: mainEventName, event: eventName, + posted: Date.now(), data, }; parentPort.postMessage(workerEvent); @@ -59,32 +67,36 @@ export function wireEventsOnWorkerThread( } } -export function wireEventsOnMainThread( +export function wireEventsOnMainThread( mainEventName: string, - events: StrictEventEmitterSingleArg, + events: StrictEventEmitterSingleArg, worker: Pick, metrics: Metrics | null, - isWorkerToMain: {[K in keyof EventData]: EventDirection} + isWorkerToMain: {[K in keyof E]: EventDirection} ): void { // Subscribe to events from main thread - worker.on("message", (data: WorkerBridgeEvent) => { + worker.on("message", (data: WorkerBridgeEvent) => { if ( typeof data === "object" && data.type === mainEventName && // This check is not necessary but added for safety in case of improper implemented events isWorkerToMain[data.event] === EventDirection.workerToMain ) { + const emitted = Date.now(); events.emit(data.event, data.data); + metrics?.networkWorkerWireEventsOnMainThreadPortLatency.observe(emitted - data.posted); + metrics?.networkWorkerWireEventsOnMainThreadEventLatency.observe(emitted - data.data.emittedAt); } }); - for (const eventName of Object.keys(isWorkerToMain) as (keyof EventData)[]) { + for (const eventName of Object.keys(isWorkerToMain) as (keyof E)[]) { if (isWorkerToMain[eventName] === EventDirection.mainToWorker) { // Pick one of the events to comply with StrictEventEmitter function signature events.on(eventName, (data) => { - const workerEvent: WorkerBridgeEvent = { + const workerEvent: WorkerBridgeEvent = { type: mainEventName, event: eventName, + posted: Date.now(), data, }; worker.postMessage(workerEvent); diff --git a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts index 000e48dfcf3a..9e83ace288be 100644 --- a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts +++ b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts @@ -126,6 +126,7 @@ describe("network / peers / PeerManager", function () { // We get a ping by peer1, don't have it's metadata so it gets requested networkEventBus.emit(NetworkEvent.reqRespRequest, { + emittedAt: Date.now(), request: {method: ReqRespMethod.Ping, body: seqNumber}, peer: peerId1, }); @@ -139,6 +140,7 @@ describe("network / peers / PeerManager", function () { // We get another ping by peer1, but with an already known seqNumber reqResp.sendMetadata.reset(); networkEventBus.emit(NetworkEvent.reqRespRequest, { + emittedAt: Date.now(), request: {method: ReqRespMethod.Ping, body: seqNumber}, peer: peerId1, }); @@ -164,6 +166,7 @@ describe("network / peers / PeerManager", function () { // Send the local status and remote status, which always passes the assertPeerRelevance function const remoteStatus = statusCache.get(); networkEventBus.emit(NetworkEvent.reqRespRequest, { + emittedAt: Date.now(), request: {method: ReqRespMethod.Status, body: remoteStatus}, peer: peerId1, }); diff --git a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts index 95ebcaa955fb..c91af52217cc 100644 --- a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts +++ b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts @@ -125,6 +125,7 @@ describe("sync / unknown block sync", function () { if (e instanceof BlockError && e.type.code === BlockErrorCode.PARENT_UNKNOWN) { // Expected bn2.network.events.emit(NetworkEvent.unknownBlockParent, { + emittedAt: Date.now(), blockInput: headInput, peer: bn2.network.peerId.toString(), }); @@ -135,6 +136,7 @@ describe("sync / unknown block sync", function () { break; case NetworkEvent.unknownBlock: bn2.network.events.emit(NetworkEvent.unknownBlock, { + emittedAt: Date.now(), rootHex: headSummary.blockRoot, peer: bn2.network.peerId.toString(), }); diff --git a/packages/beacon-node/test/unit/sync/unknownBlock.test.ts b/packages/beacon-node/test/unit/sync/unknownBlock.test.ts index 40e10fed51d4..717e849ca02d 100644 --- a/packages/beacon-node/test/unit/sync/unknownBlock.test.ts +++ b/packages/beacon-node/test/unit/sync/unknownBlock.test.ts @@ -185,11 +185,12 @@ describe("sync by UnknownBlockSync", () => { syncService.subscribeToNetwork(); if (event === NetworkEvent.unknownBlockParent) { network.events?.emit(NetworkEvent.unknownBlockParent, { + emittedAt: Date.now(), blockInput: getBlockInput.preDeneb(config, blockC, BlockSource.gossip, null), peer, }); } else { - network.events?.emit(NetworkEvent.unknownBlock, {rootHex: blockRootHexC, peer}); + network.events?.emit(NetworkEvent.unknownBlock, {emittedAt: Date.now(), rootHex: blockRootHexC, peer}); } if (wrongBlockRoot) { From 98bef5e951d5cc168c357a50ce9b8f8a572296ba Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 05:49:25 -0400 Subject: [PATCH 05/42] fix(beacon-node): move async iterator timestamps to correct location --- packages/beacon-node/src/network/core/events.ts | 8 ++++---- packages/beacon-node/src/util/asyncIterableToEvents.ts | 4 ++++ 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/src/network/core/events.ts b/packages/beacon-node/src/network/core/events.ts index ec9cd2711335..31bb0cb5d891 100644 --- a/packages/beacon-node/src/network/core/events.ts +++ b/packages/beacon-node/src/network/core/events.ts @@ -35,8 +35,8 @@ export function getReqRespBridgeReqEvents( events: IReqRespBridgeEventBus ): AsyncIterableEventBus { return { - emitRequest: (data) => events.emit(ReqRespBridgeEvent.outgoingRequest, {...data, emittedAt: Date.now()}), - emitResponse: (data) => events.emit(ReqRespBridgeEvent.incomingResponse, {...data, emittedAt: Date.now()}), + emitRequest: (data) => events.emit(ReqRespBridgeEvent.outgoingRequest, data), + emitResponse: (data) => events.emit(ReqRespBridgeEvent.incomingResponse, data), onRequest: (cb) => events.on(ReqRespBridgeEvent.outgoingRequest, cb), onResponse: (cb) => events.on(ReqRespBridgeEvent.incomingResponse, cb), }; @@ -46,8 +46,8 @@ export function getReqRespBridgeRespEvents( events: IReqRespBridgeEventBus ): AsyncIterableEventBus { return { - emitRequest: (data) => events.emit(ReqRespBridgeEvent.incomingRequest, {...data, emittedAt: Date.now()}), - emitResponse: (data) => events.emit(ReqRespBridgeEvent.outgoingResponse, {...data, emittedAt: Date.now()}), + emitRequest: (data) => events.emit(ReqRespBridgeEvent.incomingRequest, data), + emitResponse: (data) => events.emit(ReqRespBridgeEvent.outgoingResponse, data), onRequest: (cb) => events.on(ReqRespBridgeEvent.incomingRequest, cb), onResponse: (cb) => events.on(ReqRespBridgeEvent.outgoingResponse, cb), }; diff --git a/packages/beacon-node/src/util/asyncIterableToEvents.ts b/packages/beacon-node/src/util/asyncIterableToEvents.ts index 339490d550df..d1de0dfd4ab0 100644 --- a/packages/beacon-node/src/util/asyncIterableToEvents.ts +++ b/packages/beacon-node/src/util/asyncIterableToEvents.ts @@ -62,6 +62,7 @@ export class AsyncIterableBridgeCaller { self.pending.set(id, req); self.events.emitRequest({ + emittedAt: Date.now(), callArgs, id, }); @@ -147,6 +148,7 @@ export class AsyncIterableBridgeHandler { for await (const item of this.handler(data.callArgs)) { this.events.emitResponse({ type: IteratorEventType.next, + emittedAt: Date.now(), id: data.id, item, }); @@ -154,11 +156,13 @@ export class AsyncIterableBridgeHandler { this.events.emitResponse({ type: IteratorEventType.done, + emittedAt: Date.now(), id: data.id, }); } catch (e) { this.events.emitResponse({ type: IteratorEventType.error, + emittedAt: Date.now(), id: data.id, error: toThreadBoundaryError(e as Error), }); From 63708a8822917d0cf1852249d7515a2efd92b99f Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 05:50:06 -0400 Subject: [PATCH 06/42] fix(beacon-node): Omit unnecessary emittedAt from parameter type --- packages/beacon-node/src/sync/unknownBlock.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/src/sync/unknownBlock.ts b/packages/beacon-node/src/sync/unknownBlock.ts index 82654f501346..3a8cdb3a8a45 100644 --- a/packages/beacon-node/src/sync/unknownBlock.ts +++ b/packages/beacon-node/src/sync/unknownBlock.ts @@ -101,7 +101,7 @@ export class UnknownBlockSync { /** * Process an unknownBlockParent event and register the block in `pendingBlocks` Map. */ - private onUnknownParent = (data: NetworkEventData[NetworkEvent.unknownBlockParent]): void => { + private onUnknownParent = (data: Omit): void => { try { this.addUnknownParent(data.blockInput, data.peer); this.triggerUnknownBlockSearch(); From baa449708fd7837068bc106a4e66d0bc2bf6abf9 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 06:37:57 -0400 Subject: [PATCH 07/42] feat(dashboards): add network worker thread metrics to dashboard --- dashboards/lodestar_networking.json | 1460 ++++++++++++++++++--------- 1 file changed, 959 insertions(+), 501 deletions(-) diff --git a/dashboards/lodestar_networking.json b/dashboards/lodestar_networking.json index 7a18f218db7b..eddf806c3616 100644 --- a/dashboards/lodestar_networking.json +++ b/dashboards/lodestar_networking.json @@ -34,7 +34,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": null, + "id": 38, "links": [ { "asDropdown": true, @@ -54,7 +54,7 @@ "liveNow": false, "panels": [ { - "collapsed": false, + "collapsed": true, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" @@ -66,552 +66,826 @@ "y": 0 }, "id": 510, - "panels": [], - "targets": [ + "panels": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "refId": "A" - } - ], - "title": "Summary", - "type": "row" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 4, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 502, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" } }, - "mappings": [], - "min": 0, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 502, - "options": { - "graph": {}, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "lodestar_peers_by_client_count", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{client}}", + "refId": "A" + } + ], + "title": "Peers by client implementation", + "type": "timeseries" }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "7.4.5", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": false, - "expr": "lodestar_peers_by_client_count", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{client}}", - "refId": "A" - } - ], - "title": "Peers by client implementation", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "thresholdsStyle": { - "mode": "off" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 328, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 328, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "pluginVersion": "8.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "lodestar_gossip_mesh_peers_by_type_count{type!~\"beacon_attestation\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Mesh peers count", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.2.2", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": false, - "expr": "lodestar_gossip_mesh_peers_by_type_count{type!~\"beacon_attestation\"}", - "format": "time_series", - "interval": "", - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "title": "Mesh peers count", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, - "thresholdsStyle": { - "mode": "off" + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 330, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 330, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true + "pluginVersion": "8.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "lodestar_gossip_peer_score_by_threshold_count", + "format": "time_series", + "interval": "", + "legendFormat": "{{threshold}}", + "refId": "A" + } + ], + "title": "Peer count per gossip score threshold", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.2.2", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": false, - "expr": "lodestar_gossip_peer_score_by_threshold_count", - "format": "time_series", - "interval": "", - "legendFormat": "{{threshold}}", - "refId": "A" - } - ], - "title": "Peer count per gossip score threshold", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [] - }, - "overrides": [ - { - "matcher": { - "id": "byName", - "options": "max" - }, - "properties": [ - { - "id": "custom.fillBelowTo", - "value": "avg" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - { - "id": "custom.fillOpacity", - "value": 12 - }, - { - "id": "color", - "value": { - "fixedColor": "semi-dark-green", - "mode": "fixed" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } - ] - }, - { - "matcher": { - "id": "byName", - "options": "avg" }, - "properties": [ + "overrides": [ { - "id": "custom.fillBelowTo", - "value": "min" + "matcher": { + "id": "byName", + "options": "max" + }, + "properties": [ + { + "id": "custom.fillBelowTo", + "value": "avg" + }, + { + "id": "custom.fillOpacity", + "value": 12 + }, + { + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "fixed" + } + } + ] }, { - "id": "custom.fillOpacity", - "value": 12 + "matcher": { + "id": "byName", + "options": "avg" + }, + "properties": [ + { + "id": "custom.fillBelowTo", + "value": "min" + }, + { + "id": "custom.fillOpacity", + "value": 12 + }, + { + "id": "custom.gradientMode", + "value": "opacity" + }, + { + "id": "color", + "value": { + "fixedColor": "super-light-green", + "mode": "fixed" + } + } + ] }, { - "id": "custom.gradientMode", - "value": "opacity" - }, - { - "id": "color", - "value": { - "fixedColor": "super-light-green", - "mode": "fixed" - } + "matcher": { + "id": "byName", + "options": "min" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] } ] }, - { - "matcher": { - "id": "byName", - "options": "min" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-blue", - "mode": "fixed" - } - } - ] - } - ] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 331, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.2.2", - "targets": [ + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 331, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "lodestar_gossip_score_avg_min_max_max", + "format": "time_series", + "interval": "", + "legendFormat": "max", + "refId": "max" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "lodestar_gossip_score_avg_min_max_avg", + "hide": false, + "interval": "", + "legendFormat": "avg", + "refId": "avg" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "lodestar_gossip_score_avg_min_max_min", + "hide": false, + "interval": "", + "legendFormat": "min", + "refId": "min" + } + ], + "title": "Gossip score avg / min / max", + "type": "timeseries" + }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": false, - "expr": "lodestar_gossip_score_avg_min_max_max", - "format": "time_series", - "interval": "", - "legendFormat": "max", - "refId": "max" + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "min": 0.01, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 70, + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "lodestar_gossip_mesh_peers_by_beacon_attestation_subnet_count", + "interval": "", + "legendFormat": "{{subnet}}", + "refId": "A" + } + ], + "title": "Attnet mesh peers per subnet", + "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": false, - "expr": "lodestar_gossip_score_avg_min_max_avg", - "hide": false, - "interval": "", - "legendFormat": "avg", - "refId": "avg" + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0.01, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 68, + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "lodestar_gossip_mesh_peers_by_type_count{type!~\"beacon_attestation\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Mesh peers", + "type": "bargauge" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": false, - "expr": "lodestar_gossip_score_avg_min_max_min", - "hide": false, - "interval": "", - "legendFormat": "min", - "refId": "min" - } - ], - "title": "Gossip score avg / min / max", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 300, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "decimals": 0, - "mappings": [], - "min": 0.01 - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 70, - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" + "pluginVersion": "8.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "count(lodestar_gossip_mesh_peers_by_beacon_attestation_subnet_count > 0)", + "interval": "", + "legendFormat": "{{subnet}}", + "refId": "A" + } ], - "fields": "", - "values": false + "title": "Attnet count with > 0 mesh peers", + "type": "timeseries" }, - "showUnfilled": true, - "text": {} - }, - "pluginVersion": "9.3.2", - "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "exemplar": false, - "expr": "lodestar_gossip_mesh_peers_by_beacon_attestation_subnet_count", - "interval": "", - "legendFormat": "{{subnet}}", - "refId": "A" - } - ], - "title": "Attnet mesh peers per subnet", - "type": "bargauge" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 298, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } }, - "mappings": [], - "min": 0.01 - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 68, - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" + "pluginVersion": "8.2.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "lodestar_gossip_topic_peers_by_type_count{type!~\"beacon_attestation\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{type}}", + "refId": "A" + } ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "text": {} - }, - "pluginVersion": "9.3.2", + "title": "Topic peers count", + "type": "timeseries" + } + ], "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "lodestar_gossip_mesh_peers_by_type_count{type!~\"beacon_attestation\"}", - "format": "time_series", - "interval": "", - "legendFormat": "{{type}}", "refId": "A" } ], - "title": "Mesh peers", - "type": "bargauge" + "title": "Summary", + "type": "row" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 }, + "id": 608, + "panels": [], + "title": "Network Worker", + "type": "row" + }, + { + "description": "Async time to emit a network event across thread", "fieldConfig": { "defaults": { "color": { @@ -624,9 +898,10 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", + "fillOpacity": 23, + "gradientMode": "opacity", "hideFrom": { + "graph": false, "legend": false, "tooltip": false, "viz": false @@ -635,9 +910,10 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "type": "linear" + "log": 2, + "type": "log" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -647,18 +923,33 @@ "mode": "off" } }, - "mappings": [] + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 0, - "y": 23 + "y": 2 }, - "id": 300, + "id": 148, "options": { + "graph": {}, "legend": { "calcs": [], "displayMode": "list", @@ -666,11 +957,11 @@ "showLegend": true }, "tooltip": { - "mode": "single", + "mode": "multi", "sort": "none" } }, - "pluginVersion": "8.2.2", + "pluginVersion": "7.4.5", "targets": [ { "datasource": { @@ -678,20 +969,29 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "count(lodestar_gossip_mesh_peers_by_beacon_attestation_subnet_count > 0)", + "expr": "rate(network_worker_wire_events_on_worker_thread_event_latency[6m])", "interval": "", - "legendFormat": "{{subnet}}", - "refId": "A" + "legendFormat": "to worker thread", + "refId": "to worker thread" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_event_latency[6m])", + "hide": false, + "interval": "", + "legendFormat": "to main thread", + "refId": "to main thread" } ], - "title": "Attnet count with > 0 mesh peers", + "title": "Network worker - latency to re-emit network event across worker boundary", "type": "timeseries" }, { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, + "description": "Async time to send a message across the worker/parent port", "fieldConfig": { "defaults": { "color": { @@ -704,9 +1004,10 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", + "fillOpacity": 23, + "gradientMode": "opacity", "hideFrom": { + "graph": false, "legend": false, "tooltip": false, "viz": false @@ -715,9 +1016,10 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "type": "linear" + "log": 2, + "type": "log" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -727,18 +1029,33 @@ "mode": "off" } }, - "mappings": [] + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 7, + "h": 6, "w": 12, "x": 12, - "y": 23 + "y": 2 }, - "id": 298, + "id": 148, "options": { + "graph": {}, "legend": { "calcs": [], "displayMode": "list", @@ -750,7 +1067,7 @@ "sort": "none" } }, - "pluginVersion": "8.2.2", + "pluginVersion": "7.4.5", "targets": [ { "datasource": { @@ -758,14 +1075,25 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "lodestar_gossip_topic_peers_by_type_count{type!~\"beacon_attestation\"}", - "format": "time_series", + "expr": "rate(network_worker_wire_events_on_worker_thread_port_latency[6m])", "interval": "", - "legendFormat": "{{type}}", - "refId": "A" + "legendFormat": "to worker thread", + "refId": "to worker thread" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_port_latency[6m])", + "hide": false, + "interval": "", + "legendFormat": "to main thread", + "refId": "to main thread" } ], - "title": "Topic peers count", + "title": "Network worker - to/from worker port latency", "type": "timeseries" }, { @@ -1156,6 +1484,19 @@ }, "mappings": [], "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "short" }, "overrides": [] @@ -1241,6 +1582,19 @@ }, "mappings": [], "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "short" }, "overrides": [] @@ -1329,6 +1683,19 @@ } }, "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "short" }, "overrides": [] @@ -1526,6 +1893,19 @@ }, "mappings": [], "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "short" }, "overrides": [] @@ -1835,6 +2215,19 @@ } }, "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "short" }, "overrides": [ @@ -2032,6 +2425,19 @@ } }, "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "short" }, "overrides": [] @@ -2653,6 +3059,19 @@ } }, "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "percentunit" }, "overrides": [] @@ -2736,6 +3155,19 @@ } }, "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "none" }, "overrides": [] @@ -2819,6 +3251,19 @@ } }, "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "s" }, "overrides": [] @@ -2903,6 +3348,19 @@ } }, "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, "unit": "percentunit" }, "overrides": [] @@ -6310,7 +6768,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "unstable-lg1k-hzax41" + "value": "feat1-lg1k-hzax41" } ], "hide": 0, @@ -6341,6 +6799,6 @@ "timezone": "utc", "title": "Lodestar - networking", "uid": "lodestar_networking", - "version": 14, + "version": 38, "weekStart": "monday" -} +} \ No newline at end of file From c6f89f3c6cf83e7f092ec8ef7c32c57416a1b8e9 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 06:44:08 -0400 Subject: [PATCH 08/42] fix(dashboard): change metric name back to lodestar_ prefix --- dashboards/lodestar_networking.json | 4 ++-- packages/beacon-node/src/network/core/metrics.ts | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dashboards/lodestar_networking.json b/dashboards/lodestar_networking.json index eddf806c3616..4fa79315d5e2 100644 --- a/dashboards/lodestar_networking.json +++ b/dashboards/lodestar_networking.json @@ -969,7 +969,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(network_worker_wire_events_on_worker_thread_event_latency[6m])", + "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_event_latency[6m])", "interval": "", "legendFormat": "to worker thread", "refId": "to worker thread" @@ -1075,7 +1075,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(network_worker_wire_events_on_worker_thread_port_latency[6m])", + "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_port_latency[6m])", "interval": "", "legendFormat": "to worker thread", "refId": "to worker thread" diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts index b4e66e74daf3..e446025d5c54 100644 --- a/packages/beacon-node/src/network/core/metrics.ts +++ b/packages/beacon-node/src/network/core/metrics.ts @@ -339,16 +339,16 @@ export type NetworkCoreWorkerMetrics = ReturnType Date: Tue, 25 Jul 2023 06:52:09 -0400 Subject: [PATCH 09/42] fix: run check-types and update missed test types --- .../onWorker/dataSerialization.test.ts | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts b/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts index 517359b93efd..08d60cfb692d 100644 --- a/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts +++ b/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts @@ -44,14 +44,24 @@ describe("data serialization through worker boundary", function () { // Defining tests in this notation ensures that any event data is tested and probably safe to send const reqRespBridgeEventData: ReqRespBridgeEventData = { - [ReqRespBridgeEvent.outgoingRequest]: {id: 0, callArgs: {peerId, method, versions: [1, 2], requestData: bytes}}, + [ReqRespBridgeEvent.outgoingRequest]: { + emittedAt: Date.now(), + id: 0, + callArgs: {peerId, method, versions: [1, 2], requestData: bytes}, + }, [ReqRespBridgeEvent.outgoingResponse]: { + emittedAt: Date.now(), type: IteratorEventType.next, id: 0, item: {data: bytes, fork: ForkName.altair}, }, - [ReqRespBridgeEvent.incomingRequest]: {id: 0, callArgs: {method, req: {data: bytes, version: 1}, peerId}}, + [ReqRespBridgeEvent.incomingRequest]: { + emittedAt: Date.now(), + id: 0, + callArgs: {method, req: {data: bytes, version: 1}, peerId}, + }, [ReqRespBridgeEvent.incomingResponse]: { + emittedAt: Date.now(), type: IteratorEventType.next, id: 0, item: {data: bytes, fork: ForkName.altair, protocolVersion: 1}, @@ -73,13 +83,15 @@ describe("data serialization through worker boundary", function () { // Defining tests in this notation ensures that any event data is tested and probably safe to send const networkEventData = filterByUsedEvents(networkEventDirection, { - [NetworkEvent.peerConnected]: {peer, status: statusZero}, - [NetworkEvent.peerDisconnected]: {peer}, + [NetworkEvent.peerConnected]: {emittedAt: Date.now(), peer, status: statusZero}, + [NetworkEvent.peerDisconnected]: {emittedAt: Date.now(), peer}, [NetworkEvent.reqRespRequest]: { + emittedAt: Date.now(), request: {method: ReqRespMethod.Status, body: statusZero}, peer: getValidPeerId(), }, [NetworkEvent.unknownBlockParent]: { + emittedAt: Date.now(), blockInput: { type: BlockInputType.preDeneb, block: ssz.capella.SignedBeaconBlock.defaultValue(), @@ -88,11 +100,9 @@ describe("data serialization through worker boundary", function () { }, peer, }, - [NetworkEvent.unknownBlock]: { - rootHex: ZERO_HASH_HEX, - peer, - }, + [NetworkEvent.unknownBlock]: {emittedAt: Date.now(), rootHex: ZERO_HASH_HEX, peer}, [NetworkEvent.pendingGossipsubMessage]: { + emittedAt: Date.now(), topic: {type: GossipType.beacon_block, fork: ForkName.altair}, msg: { type: "unsigned", @@ -106,6 +116,7 @@ describe("data serialization through worker boundary", function () { startProcessUnixSec: 1600000000, }, [NetworkEvent.gossipMessageValidationResult]: { + emittedAt: Date.now(), msgId: ZERO_HASH_HEX, propagationSource: peerId, acceptance: TopicValidatorResult.Accept, From 37a097b863c458e7ae5623f70befc39fba6e7d8d Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 07:03:18 -0400 Subject: [PATCH 10/42] Revert "feat(beacon-node): add types for network worker event metrics" This reverts commit e8dc6d29458d5c54c02a71c01f390f34118605ab. --- packages/beacon-node/src/network/events.ts | 15 +++++++-------- .../beacon-node/src/util/asyncIterableToEvents.ts | 9 ++++----- packages/beacon-node/src/util/types.ts | 2 -- 3 files changed, 11 insertions(+), 15 deletions(-) diff --git a/packages/beacon-node/src/network/events.ts b/packages/beacon-node/src/network/events.ts index 5520287e8eac..815efda7d9b8 100644 --- a/packages/beacon-node/src/network/events.ts +++ b/packages/beacon-node/src/network/events.ts @@ -6,7 +6,6 @@ import {BlockInput} from "../chain/blocks/types.js"; import {StrictEventEmitterSingleArg} from "../util/strictEvents.js"; import {PeerIdStr} from "../util/peerId.js"; import {EventDirection} from "../util/workerEvents.js"; -import {EmittedAt} from "../util/types.js"; import {RequestTypedContainer} from "./reqresp/ReqRespBeaconNode.js"; import {PendingGossipsubMessage} from "./processor/types.js"; @@ -28,13 +27,13 @@ export enum NetworkEvent { } export type NetworkEventData = { - [NetworkEvent.peerConnected]: EmittedAt & {peer: PeerIdStr; status: phase0.Status}; - [NetworkEvent.peerDisconnected]: EmittedAt & {peer: PeerIdStr}; - [NetworkEvent.reqRespRequest]: EmittedAt & {request: RequestTypedContainer; peer: PeerId}; - [NetworkEvent.unknownBlockParent]: EmittedAt & {blockInput: BlockInput; peer: PeerIdStr}; - [NetworkEvent.unknownBlock]: EmittedAt & {rootHex: RootHex; peer?: PeerIdStr}; - [NetworkEvent.pendingGossipsubMessage]: EmittedAt & PendingGossipsubMessage; - [NetworkEvent.gossipMessageValidationResult]: EmittedAt & { + [NetworkEvent.peerConnected]: {peer: PeerIdStr; status: phase0.Status}; + [NetworkEvent.peerDisconnected]: {peer: PeerIdStr}; + [NetworkEvent.reqRespRequest]: {request: RequestTypedContainer; peer: PeerId}; + [NetworkEvent.unknownBlockParent]: {blockInput: BlockInput; peer: PeerIdStr}; + [NetworkEvent.unknownBlock]: {rootHex: RootHex; peer?: PeerIdStr}; + [NetworkEvent.pendingGossipsubMessage]: PendingGossipsubMessage; + [NetworkEvent.gossipMessageValidationResult]: { msgId: string; propagationSource: PeerIdStr; acceptance: TopicValidatorResult; diff --git a/packages/beacon-node/src/util/asyncIterableToEvents.ts b/packages/beacon-node/src/util/asyncIterableToEvents.ts index d1de0dfd4ab0..2530b668b843 100644 --- a/packages/beacon-node/src/util/asyncIterableToEvents.ts +++ b/packages/beacon-node/src/util/asyncIterableToEvents.ts @@ -1,8 +1,7 @@ import {LinkedList} from "./array.js"; import {ThreadBoundaryError, fromThreadBoundaryError, toThreadBoundaryError} from "./error.js"; -import {EmittedAt} from "./types.js"; -export type RequestEvent = EmittedAt & { +export type RequestEvent = { callArgs: T; id: number; }; @@ -14,9 +13,9 @@ export enum IteratorEventType { } export type IteratorEvent = - | (EmittedAt & {type: IteratorEventType.next; id: number; item: V}) - | (EmittedAt & {type: IteratorEventType.done; id: number}) - | (EmittedAt & {type: IteratorEventType.error; id: number; error: ThreadBoundaryError}); + | {type: IteratorEventType.next; id: number; item: V} + | {type: IteratorEventType.done; id: number} + | {type: IteratorEventType.error; id: number; error: ThreadBoundaryError}; export type AsyncIterableEventBus = { emitRequest(data: RequestEvent): void; diff --git a/packages/beacon-node/src/util/types.ts b/packages/beacon-node/src/util/types.ts index 815d49e20835..545a706c7511 100644 --- a/packages/beacon-node/src/util/types.ts +++ b/packages/beacon-node/src/util/types.ts @@ -12,5 +12,3 @@ export const signedBLSToExecutionChangeVersionedType = new ContainerType( {jsonCase: "eth2", typeName: "SignedBLSToExecutionChangeVersionedType"} ); export type SignedBLSToExecutionChangeVersioned = ValueOf; - -export type EmittedAt = {emittedAt: number}; From 8a205d11bdd79d6a97b0aa68174a762779455846 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 07:03:47 -0400 Subject: [PATCH 11/42] Revert "fix(beacon-node): move async iterator timestamps to correct location" This reverts commit 8b3f6c606e0857e94275daa25381b77c4e8c8202. --- packages/beacon-node/src/network/core/events.ts | 8 ++++---- packages/beacon-node/src/util/asyncIterableToEvents.ts | 4 ---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/beacon-node/src/network/core/events.ts b/packages/beacon-node/src/network/core/events.ts index 31bb0cb5d891..ec9cd2711335 100644 --- a/packages/beacon-node/src/network/core/events.ts +++ b/packages/beacon-node/src/network/core/events.ts @@ -35,8 +35,8 @@ export function getReqRespBridgeReqEvents( events: IReqRespBridgeEventBus ): AsyncIterableEventBus { return { - emitRequest: (data) => events.emit(ReqRespBridgeEvent.outgoingRequest, data), - emitResponse: (data) => events.emit(ReqRespBridgeEvent.incomingResponse, data), + emitRequest: (data) => events.emit(ReqRespBridgeEvent.outgoingRequest, {...data, emittedAt: Date.now()}), + emitResponse: (data) => events.emit(ReqRespBridgeEvent.incomingResponse, {...data, emittedAt: Date.now()}), onRequest: (cb) => events.on(ReqRespBridgeEvent.outgoingRequest, cb), onResponse: (cb) => events.on(ReqRespBridgeEvent.incomingResponse, cb), }; @@ -46,8 +46,8 @@ export function getReqRespBridgeRespEvents( events: IReqRespBridgeEventBus ): AsyncIterableEventBus { return { - emitRequest: (data) => events.emit(ReqRespBridgeEvent.incomingRequest, data), - emitResponse: (data) => events.emit(ReqRespBridgeEvent.outgoingResponse, data), + emitRequest: (data) => events.emit(ReqRespBridgeEvent.incomingRequest, {...data, emittedAt: Date.now()}), + emitResponse: (data) => events.emit(ReqRespBridgeEvent.outgoingResponse, {...data, emittedAt: Date.now()}), onRequest: (cb) => events.on(ReqRespBridgeEvent.incomingRequest, cb), onResponse: (cb) => events.on(ReqRespBridgeEvent.outgoingResponse, cb), }; diff --git a/packages/beacon-node/src/util/asyncIterableToEvents.ts b/packages/beacon-node/src/util/asyncIterableToEvents.ts index 2530b668b843..0468ca2b431b 100644 --- a/packages/beacon-node/src/util/asyncIterableToEvents.ts +++ b/packages/beacon-node/src/util/asyncIterableToEvents.ts @@ -61,7 +61,6 @@ export class AsyncIterableBridgeCaller { self.pending.set(id, req); self.events.emitRequest({ - emittedAt: Date.now(), callArgs, id, }); @@ -147,7 +146,6 @@ export class AsyncIterableBridgeHandler { for await (const item of this.handler(data.callArgs)) { this.events.emitResponse({ type: IteratorEventType.next, - emittedAt: Date.now(), id: data.id, item, }); @@ -155,13 +153,11 @@ export class AsyncIterableBridgeHandler { this.events.emitResponse({ type: IteratorEventType.done, - emittedAt: Date.now(), id: data.id, }); } catch (e) { this.events.emitResponse({ type: IteratorEventType.error, - emittedAt: Date.now(), id: data.id, error: toThreadBoundaryError(e as Error), }); From 78714cc74bcac7ee013c297b9f14741ee6d467a6 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 07:04:00 -0400 Subject: [PATCH 12/42] Revert "fix(beacon-node): Omit unnecessary emittedAt from parameter type" This reverts commit b7fde56b5f903cc767a84a9f5ce861d867c8b480. --- packages/beacon-node/src/sync/unknownBlock.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/src/sync/unknownBlock.ts b/packages/beacon-node/src/sync/unknownBlock.ts index 3a8cdb3a8a45..82654f501346 100644 --- a/packages/beacon-node/src/sync/unknownBlock.ts +++ b/packages/beacon-node/src/sync/unknownBlock.ts @@ -101,7 +101,7 @@ export class UnknownBlockSync { /** * Process an unknownBlockParent event and register the block in `pendingBlocks` Map. */ - private onUnknownParent = (data: Omit): void => { + private onUnknownParent = (data: NetworkEventData[NetworkEvent.unknownBlockParent]): void => { try { this.addUnknownParent(data.blockInput, data.peer); this.triggerUnknownBlockSearch(); From 4d49479e85f819d4b724cb66cda037e074adea1f Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 07:06:51 -0400 Subject: [PATCH 13/42] Revert "feat(beacon-node): add metric data to network worker events" This reverts commit 631f57071ebf5a5595f2c0d02c8ed365825ced61. --- .../beacon-node/src/network/core/events.ts | 8 ++-- .../src/network/gossip/gossipsub.ts | 1 - .../src/network/peers/peerManager.ts | 2 +- .../src/network/processor/gossipHandlers.ts | 2 +- .../src/network/processor/index.ts | 2 +- .../src/network/reqresp/ReqRespBeaconNode.ts | 2 +- packages/beacon-node/src/util/workerEvents.ts | 38 +++++++------------ .../e2e/network/peers/peerManager.test.ts | 3 -- .../test/e2e/sync/unknownBlockSync.test.ts | 2 - .../test/unit/sync/unknownBlock.test.ts | 3 +- 10 files changed, 22 insertions(+), 41 deletions(-) diff --git a/packages/beacon-node/src/network/core/events.ts b/packages/beacon-node/src/network/core/events.ts index ec9cd2711335..31bb0cb5d891 100644 --- a/packages/beacon-node/src/network/core/events.ts +++ b/packages/beacon-node/src/network/core/events.ts @@ -35,8 +35,8 @@ export function getReqRespBridgeReqEvents( events: IReqRespBridgeEventBus ): AsyncIterableEventBus { return { - emitRequest: (data) => events.emit(ReqRespBridgeEvent.outgoingRequest, {...data, emittedAt: Date.now()}), - emitResponse: (data) => events.emit(ReqRespBridgeEvent.incomingResponse, {...data, emittedAt: Date.now()}), + emitRequest: (data) => events.emit(ReqRespBridgeEvent.outgoingRequest, data), + emitResponse: (data) => events.emit(ReqRespBridgeEvent.incomingResponse, data), onRequest: (cb) => events.on(ReqRespBridgeEvent.outgoingRequest, cb), onResponse: (cb) => events.on(ReqRespBridgeEvent.incomingResponse, cb), }; @@ -46,8 +46,8 @@ export function getReqRespBridgeRespEvents( events: IReqRespBridgeEventBus ): AsyncIterableEventBus { return { - emitRequest: (data) => events.emit(ReqRespBridgeEvent.incomingRequest, {...data, emittedAt: Date.now()}), - emitResponse: (data) => events.emit(ReqRespBridgeEvent.outgoingResponse, {...data, emittedAt: Date.now()}), + emitRequest: (data) => events.emit(ReqRespBridgeEvent.incomingRequest, data), + emitResponse: (data) => events.emit(ReqRespBridgeEvent.outgoingResponse, data), onRequest: (cb) => events.on(ReqRespBridgeEvent.incomingRequest, cb), onResponse: (cb) => events.on(ReqRespBridgeEvent.outgoingResponse, cb), }; diff --git a/packages/beacon-node/src/network/gossip/gossipsub.ts b/packages/beacon-node/src/network/gossip/gossipsub.ts index 65e9b01bc2d1..649d8a0f9361 100644 --- a/packages/beacon-node/src/network/gossip/gossipsub.ts +++ b/packages/beacon-node/src/network/gossip/gossipsub.ts @@ -283,7 +283,6 @@ export class Eth2Gossipsub extends GossipSub { // See https://github.com/ChainSafe/lodestar/issues/5604 setTimeout(() => { this.events.emit(NetworkEvent.pendingGossipsubMessage, { - emittedAt: Date.now(), topic, msg, msgId, diff --git a/packages/beacon-node/src/network/peers/peerManager.ts b/packages/beacon-node/src/network/peers/peerManager.ts index 5b6df685066c..0ce3fdbb107c 100644 --- a/packages/beacon-node/src/network/peers/peerManager.ts +++ b/packages/beacon-node/src/network/peers/peerManager.ts @@ -377,7 +377,7 @@ export class PeerManager { peerData.relevantStatus = RelevantPeerStatus.relevant; } if (getConnection(this.libp2p, peer.toString())) { - this.networkEventBus.emit(NetworkEvent.peerConnected, {emittedAt: Date.now(), peer: peer.toString(), status}); + this.networkEventBus.emit(NetworkEvent.peerConnected, {peer: peer.toString(), status}); } } diff --git a/packages/beacon-node/src/network/processor/gossipHandlers.ts b/packages/beacon-node/src/network/processor/gossipHandlers.ts index a09eefa597fa..4916b98abfee 100644 --- a/packages/beacon-node/src/network/processor/gossipHandlers.ts +++ b/packages/beacon-node/src/network/processor/gossipHandlers.ts @@ -150,7 +150,7 @@ function getDefaultHandlers(modules: ValidatorFnsModules, options: GossipHandler // Don't trigger this yet if full block and blobs haven't arrived yet if (e instanceof BlockGossipError && e.type.code === BlockErrorCode.PARENT_UNKNOWN && blockInput !== null) { logger.debug("Gossip block has error", {slot, root: blockHex, code: e.type.code}); - events.emit(NetworkEvent.unknownBlockParent, {emittedAt: Date.now(), blockInput, peer: peerIdStr}); + events.emit(NetworkEvent.unknownBlockParent, {blockInput, peer: peerIdStr}); } } diff --git a/packages/beacon-node/src/network/processor/index.ts b/packages/beacon-node/src/network/processor/index.ts index ce13de72a220..4abf4baa7472 100644 --- a/packages/beacon-node/src/network/processor/index.ts +++ b/packages/beacon-node/src/network/processor/index.ts @@ -234,7 +234,7 @@ export class NetworkProcessor { // Search for the unknown block if (!this.unknownRootsBySlot.getOrDefault(slot).has(root)) { this.unknownRootsBySlot.getOrDefault(slot).add(root); - this.events.emit(NetworkEvent.unknownBlock, {emittedAt: Date.now(), rootHex: root, peer}); + this.events.emit(NetworkEvent.unknownBlock, {rootHex: root, peer}); } } diff --git a/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts b/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts index 19a8678c0c1f..69b83ee327c6 100644 --- a/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts +++ b/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts @@ -259,7 +259,7 @@ export class ReqRespBeaconNode extends ReqResp { // Allow onRequest to return and close the stream // For Goodbye there may be a race condition where the listener of `receivedGoodbye` // disconnects in the same syncronous call, preventing the stream from ending cleanly - setTimeout(() => this.networkEventBus.emit(NetworkEvent.reqRespRequest, {emittedAt: Date.now(), request, peer}), 0); + setTimeout(() => this.networkEventBus.emit(NetworkEvent.reqRespRequest, {request, peer}), 0); } protected onIncomingRequest(peerId: PeerId, protocol: ProtocolDescriptor): void { diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 8bc1f79852ff..02d2ff47fba2 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -5,15 +5,11 @@ import {sleep} from "@lodestar/utils"; import {Metrics} from "../metrics/metrics.js"; import {NetworkCoreWorkerMetrics} from "../network/core/metrics.js"; import {StrictEventEmitterSingleArg} from "./strictEvents.js"; -import {EmittedAt} from "./types.js"; -type EventData = {[key: string]: EmittedAt}; - -export type WorkerBridgeEvent = { +export type WorkerBridgeEvent = { type: string; - event: keyof E; - posted: number; - data: E[keyof E]; + event: keyof EventData; + data: EventData[keyof EventData]; }; export enum EventDirection { @@ -29,25 +25,22 @@ export enum EventDirection { * - worker to main * - main to worker */ -export function wireEventsOnWorkerThread( +export function wireEventsOnWorkerThread( mainEventName: string, - events: StrictEventEmitterSingleArg, + events: StrictEventEmitterSingleArg, parentPort: MessagePort, metrics: NetworkCoreWorkerMetrics | null, - isWorkerToMain: {[K in keyof E]: EventDirection} + isWorkerToMain: {[K in keyof EventData]: EventDirection} ): void { // Subscribe to events from main thread - parentPort.on("message", (data: WorkerBridgeEvent) => { + parentPort.on("message", (data: WorkerBridgeEvent) => { if ( typeof data === "object" && data.type === mainEventName && // This check is not necessary but added for safety in case of improper implemented events isWorkerToMain[data.event] === EventDirection.mainToWorker ) { - const emitted = Date.now(); events.emit(data.event, data.data); - metrics?.networkWorkerWireEventsOnWorkerThreadPortLatency.observe(emitted - data.posted); - metrics?.networkWorkerWireEventsOnWorkerThreadEventLatency.observe(emitted - data.data.emittedAt); } }); @@ -58,7 +51,6 @@ export function wireEventsOnWorkerThread( const workerEvent: WorkerBridgeEvent = { type: mainEventName, event: eventName, - posted: Date.now(), data, }; parentPort.postMessage(workerEvent); @@ -67,36 +59,32 @@ export function wireEventsOnWorkerThread( } } -export function wireEventsOnMainThread( +export function wireEventsOnMainThread( mainEventName: string, - events: StrictEventEmitterSingleArg, + events: StrictEventEmitterSingleArg, worker: Pick, metrics: Metrics | null, - isWorkerToMain: {[K in keyof E]: EventDirection} + isWorkerToMain: {[K in keyof EventData]: EventDirection} ): void { // Subscribe to events from main thread - worker.on("message", (data: WorkerBridgeEvent) => { + worker.on("message", (data: WorkerBridgeEvent) => { if ( typeof data === "object" && data.type === mainEventName && // This check is not necessary but added for safety in case of improper implemented events isWorkerToMain[data.event] === EventDirection.workerToMain ) { - const emitted = Date.now(); events.emit(data.event, data.data); - metrics?.networkWorkerWireEventsOnMainThreadPortLatency.observe(emitted - data.posted); - metrics?.networkWorkerWireEventsOnMainThreadEventLatency.observe(emitted - data.data.emittedAt); } }); - for (const eventName of Object.keys(isWorkerToMain) as (keyof E)[]) { + for (const eventName of Object.keys(isWorkerToMain) as (keyof EventData)[]) { if (isWorkerToMain[eventName] === EventDirection.mainToWorker) { // Pick one of the events to comply with StrictEventEmitter function signature events.on(eventName, (data) => { - const workerEvent: WorkerBridgeEvent = { + const workerEvent: WorkerBridgeEvent = { type: mainEventName, event: eventName, - posted: Date.now(), data, }; worker.postMessage(workerEvent); diff --git a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts index 9e83ace288be..000e48dfcf3a 100644 --- a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts +++ b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts @@ -126,7 +126,6 @@ describe("network / peers / PeerManager", function () { // We get a ping by peer1, don't have it's metadata so it gets requested networkEventBus.emit(NetworkEvent.reqRespRequest, { - emittedAt: Date.now(), request: {method: ReqRespMethod.Ping, body: seqNumber}, peer: peerId1, }); @@ -140,7 +139,6 @@ describe("network / peers / PeerManager", function () { // We get another ping by peer1, but with an already known seqNumber reqResp.sendMetadata.reset(); networkEventBus.emit(NetworkEvent.reqRespRequest, { - emittedAt: Date.now(), request: {method: ReqRespMethod.Ping, body: seqNumber}, peer: peerId1, }); @@ -166,7 +164,6 @@ describe("network / peers / PeerManager", function () { // Send the local status and remote status, which always passes the assertPeerRelevance function const remoteStatus = statusCache.get(); networkEventBus.emit(NetworkEvent.reqRespRequest, { - emittedAt: Date.now(), request: {method: ReqRespMethod.Status, body: remoteStatus}, peer: peerId1, }); diff --git a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts index c91af52217cc..95ebcaa955fb 100644 --- a/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts +++ b/packages/beacon-node/test/e2e/sync/unknownBlockSync.test.ts @@ -125,7 +125,6 @@ describe("sync / unknown block sync", function () { if (e instanceof BlockError && e.type.code === BlockErrorCode.PARENT_UNKNOWN) { // Expected bn2.network.events.emit(NetworkEvent.unknownBlockParent, { - emittedAt: Date.now(), blockInput: headInput, peer: bn2.network.peerId.toString(), }); @@ -136,7 +135,6 @@ describe("sync / unknown block sync", function () { break; case NetworkEvent.unknownBlock: bn2.network.events.emit(NetworkEvent.unknownBlock, { - emittedAt: Date.now(), rootHex: headSummary.blockRoot, peer: bn2.network.peerId.toString(), }); diff --git a/packages/beacon-node/test/unit/sync/unknownBlock.test.ts b/packages/beacon-node/test/unit/sync/unknownBlock.test.ts index 717e849ca02d..40e10fed51d4 100644 --- a/packages/beacon-node/test/unit/sync/unknownBlock.test.ts +++ b/packages/beacon-node/test/unit/sync/unknownBlock.test.ts @@ -185,12 +185,11 @@ describe("sync by UnknownBlockSync", () => { syncService.subscribeToNetwork(); if (event === NetworkEvent.unknownBlockParent) { network.events?.emit(NetworkEvent.unknownBlockParent, { - emittedAt: Date.now(), blockInput: getBlockInput.preDeneb(config, blockC, BlockSource.gossip, null), peer, }); } else { - network.events?.emit(NetworkEvent.unknownBlock, {emittedAt: Date.now(), rootHex: blockRootHexC, peer}); + network.events?.emit(NetworkEvent.unknownBlock, {rootHex: blockRootHexC, peer}); } if (wrongBlockRoot) { From 1bb9fbd1c62cfcbe113e0be474f664ca1d0ca906 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 07:20:38 -0400 Subject: [PATCH 14/42] feat(beacon-node): capture worker message in hrTime --- packages/beacon-node/src/util/workerEvents.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 02d2ff47fba2..07627e9e71c4 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -9,6 +9,7 @@ import {StrictEventEmitterSingleArg} from "./strictEvents.js"; export type WorkerBridgeEvent = { type: string; event: keyof EventData; + posted: [number, number]; data: EventData[keyof EventData]; }; @@ -41,6 +42,8 @@ export function wireEventsOnWorkerThread( isWorkerToMain[data.event] === EventDirection.mainToWorker ) { events.emit(data.event, data.data); + const [sec, nanoSec] = process.hrtime(data.posted); + metrics?.networkWorkerWireEventsOnWorkerThreadPortLatency.observe(sec * 1e9 + nanoSec); } }); @@ -51,6 +54,7 @@ export function wireEventsOnWorkerThread( const workerEvent: WorkerBridgeEvent = { type: mainEventName, event: eventName, + posted: process.hrtime(), data, }; parentPort.postMessage(workerEvent); @@ -75,6 +79,8 @@ export function wireEventsOnMainThread( isWorkerToMain[data.event] === EventDirection.workerToMain ) { events.emit(data.event, data.data); + const [sec, nanoSec] = process.hrtime(data.posted); + metrics?.networkWorkerWireEventsOnMainThreadPortLatency.observe(sec * 1e9 + nanoSec); } }); @@ -85,6 +91,7 @@ export function wireEventsOnMainThread( const workerEvent: WorkerBridgeEvent = { type: mainEventName, event: eventName, + posted: process.hrtime(), data, }; worker.postMessage(workerEvent); From 69f720adce899e37f3cc40ccce04c8af44f9a11d Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 07:22:36 -0400 Subject: [PATCH 15/42] fix(dashboards): remove re-emit panel from network worker row --- dashboards/lodestar_networking.json | 106 ---------------------------- 1 file changed, 106 deletions(-) diff --git a/dashboards/lodestar_networking.json b/dashboards/lodestar_networking.json index 4fa79315d5e2..d3a77bed8077 100644 --- a/dashboards/lodestar_networking.json +++ b/dashboards/lodestar_networking.json @@ -884,112 +884,6 @@ "title": "Network Worker", "type": "row" }, - { - "description": "Async time to emit a network event across thread", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 23, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "log": 2, - "type": "log" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 0, - "y": 2 - }, - "id": 148, - "options": { - "graph": {}, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } - }, - "pluginVersion": "7.4.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_event_latency[6m])", - "interval": "", - "legendFormat": "to worker thread", - "refId": "to worker thread" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_event_latency[6m])", - "hide": false, - "interval": "", - "legendFormat": "to main thread", - "refId": "to main thread" - } - ], - "title": "Network worker - latency to re-emit network event across worker boundary", - "type": "timeseries" - }, { "description": "Async time to send a message across the worker/parent port", "fieldConfig": { From f70eeb37ed61bcd91c83244da92cf300b2bd76d7 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 07:25:01 -0400 Subject: [PATCH 16/42] fix(metrics): remove unused re-emit metrics --- dashboards/lodestar_networking.json | 4 ++-- packages/beacon-node/src/metrics/metrics/lodestar.ts | 9 ++------- packages/beacon-node/src/network/core/metrics.ts | 9 ++------- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/dashboards/lodestar_networking.json b/dashboards/lodestar_networking.json index d3a77bed8077..6e5d0fc8bfb3 100644 --- a/dashboards/lodestar_networking.json +++ b/dashboards/lodestar_networking.json @@ -969,7 +969,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_port_latency[6m])", + "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency[6m])", "interval": "", "legendFormat": "to worker thread", "refId": "to worker thread" @@ -980,7 +980,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_port_latency[6m])", + "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency[6m])", "hide": false, "interval": "", "legendFormat": "to main thread", diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 5aa067b636ec..c8a86224ef40 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -121,13 +121,8 @@ export function createLodestarMetrics( help: "Current count of pending items in reqRespBridgeReqCaller data structure", }), }, - networkWorkerWireEventsOnMainThreadEventLatency: register.histogram({ - name: "lodestar_network_worker_wire_events_on_main_thread_event_latency", - help: "Latency to re-emit network events on main thread event bus", - buckets: [0.001, 0.003, 0.01, 0.03, 0.1], - }), - networkWorkerWireEventsOnMainThreadPortLatency: register.histogram({ - name: "lodestar_network_worker_wire_events_on_main_thread_port_latency", + networkWorkerWireEventsOnMainThreadLatency: register.histogram({ + name: "lodestar_network_worker_wire_events_on_main_thread_latency", help: "Latency to transmit network events to main thread across worker port", buckets: [0.001, 0.003, 0.01, 0.03, 0.1], }), diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts index e446025d5c54..edfde7c61423 100644 --- a/packages/beacon-node/src/network/core/metrics.ts +++ b/packages/beacon-node/src/network/core/metrics.ts @@ -342,13 +342,8 @@ export function getNetworkCoreWorkerMetrics(register: RegistryMetricCreator) { name: "lodestar_network_worker_reqresp_bridge_caller_pending_count", help: "Current count of pending elements in respBridgeCaller", }), - networkWorkerWireEventsOnWorkerThreadEventLatency: register.histogram({ - name: "lodestar_network_worker_wire_events_on_worker_thread_event_latency", - help: "Latency to re-emit network events on worker thread event bus", - buckets: [0.001, 0.003, 0.01, 0.03, 0.1], - }), - networkWorkerWireEventsOnWorkerThreadPortLatency: register.histogram({ - name: "lodestar_network_worker_wire_events_on_worker_thread_port_latency", + networkWorkerWireEventsOnWorkerThreadLatency: register.histogram({ + name: "lodestar_network_worker_wire_events_on_worker_thread_latency", help: "Latency to transmit network events to worker thread across parent port", buckets: [0.001, 0.003, 0.01, 0.03, 0.1], }), From 6117bd06edaf61d91cbb8869291a5c843821c125 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 07:26:56 -0400 Subject: [PATCH 17/42] Revert "fix: run check-types and update missed test types" This reverts commit 525d1577baa6ddf5050fc89f19de8fcdbeca2373. --- .../onWorker/dataSerialization.test.ts | 27 ++++++------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts b/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts index 08d60cfb692d..517359b93efd 100644 --- a/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts +++ b/packages/beacon-node/test/e2e/network/onWorker/dataSerialization.test.ts @@ -44,24 +44,14 @@ describe("data serialization through worker boundary", function () { // Defining tests in this notation ensures that any event data is tested and probably safe to send const reqRespBridgeEventData: ReqRespBridgeEventData = { - [ReqRespBridgeEvent.outgoingRequest]: { - emittedAt: Date.now(), - id: 0, - callArgs: {peerId, method, versions: [1, 2], requestData: bytes}, - }, + [ReqRespBridgeEvent.outgoingRequest]: {id: 0, callArgs: {peerId, method, versions: [1, 2], requestData: bytes}}, [ReqRespBridgeEvent.outgoingResponse]: { - emittedAt: Date.now(), type: IteratorEventType.next, id: 0, item: {data: bytes, fork: ForkName.altair}, }, - [ReqRespBridgeEvent.incomingRequest]: { - emittedAt: Date.now(), - id: 0, - callArgs: {method, req: {data: bytes, version: 1}, peerId}, - }, + [ReqRespBridgeEvent.incomingRequest]: {id: 0, callArgs: {method, req: {data: bytes, version: 1}, peerId}}, [ReqRespBridgeEvent.incomingResponse]: { - emittedAt: Date.now(), type: IteratorEventType.next, id: 0, item: {data: bytes, fork: ForkName.altair, protocolVersion: 1}, @@ -83,15 +73,13 @@ describe("data serialization through worker boundary", function () { // Defining tests in this notation ensures that any event data is tested and probably safe to send const networkEventData = filterByUsedEvents(networkEventDirection, { - [NetworkEvent.peerConnected]: {emittedAt: Date.now(), peer, status: statusZero}, - [NetworkEvent.peerDisconnected]: {emittedAt: Date.now(), peer}, + [NetworkEvent.peerConnected]: {peer, status: statusZero}, + [NetworkEvent.peerDisconnected]: {peer}, [NetworkEvent.reqRespRequest]: { - emittedAt: Date.now(), request: {method: ReqRespMethod.Status, body: statusZero}, peer: getValidPeerId(), }, [NetworkEvent.unknownBlockParent]: { - emittedAt: Date.now(), blockInput: { type: BlockInputType.preDeneb, block: ssz.capella.SignedBeaconBlock.defaultValue(), @@ -100,9 +88,11 @@ describe("data serialization through worker boundary", function () { }, peer, }, - [NetworkEvent.unknownBlock]: {emittedAt: Date.now(), rootHex: ZERO_HASH_HEX, peer}, + [NetworkEvent.unknownBlock]: { + rootHex: ZERO_HASH_HEX, + peer, + }, [NetworkEvent.pendingGossipsubMessage]: { - emittedAt: Date.now(), topic: {type: GossipType.beacon_block, fork: ForkName.altair}, msg: { type: "unsigned", @@ -116,7 +106,6 @@ describe("data serialization through worker boundary", function () { startProcessUnixSec: 1600000000, }, [NetworkEvent.gossipMessageValidationResult]: { - emittedAt: Date.now(), msgId: ZERO_HASH_HEX, propagationSource: peerId, acceptance: TopicValidatorResult.Accept, From 79c1430f214e50632660eeca097ede505537a234 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 07:29:02 -0400 Subject: [PATCH 18/42] fix(beacon-node): update metric name at call site --- packages/beacon-node/src/util/workerEvents.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 07627e9e71c4..6e2908942cbd 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -43,7 +43,7 @@ export function wireEventsOnWorkerThread( ) { events.emit(data.event, data.data); const [sec, nanoSec] = process.hrtime(data.posted); - metrics?.networkWorkerWireEventsOnWorkerThreadPortLatency.observe(sec * 1e9 + nanoSec); + metrics?.networkWorkerWireEventsOnWorkerThreadLatency.observe(sec * 1e9 + nanoSec); } }); @@ -80,7 +80,7 @@ export function wireEventsOnMainThread( ) { events.emit(data.event, data.data); const [sec, nanoSec] = process.hrtime(data.posted); - metrics?.networkWorkerWireEventsOnMainThreadPortLatency.observe(sec * 1e9 + nanoSec); + metrics?.networkWorkerWireEventsOnMainThreadLatency.observe(sec * 1e9 + nanoSec); } }); From c89014826a7216b3d8e021d3488159fc5a7bdce8 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 07:48:16 -0400 Subject: [PATCH 19/42] fix(metrics): update capture to ISU units --- packages/beacon-node/src/util/workerEvents.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 6e2908942cbd..ad18bba493e2 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -43,7 +43,7 @@ export function wireEventsOnWorkerThread( ) { events.emit(data.event, data.data); const [sec, nanoSec] = process.hrtime(data.posted); - metrics?.networkWorkerWireEventsOnWorkerThreadLatency.observe(sec * 1e9 + nanoSec); + metrics?.networkWorkerWireEventsOnWorkerThreadLatency.observe(sec + nanoSec / 1e9); } }); @@ -80,7 +80,7 @@ export function wireEventsOnMainThread( ) { events.emit(data.event, data.data); const [sec, nanoSec] = process.hrtime(data.posted); - metrics?.networkWorkerWireEventsOnMainThreadLatency.observe(sec * 1e9 + nanoSec); + metrics?.networkWorkerWireEventsOnMainThreadLatency.observe(sec + nanoSec / 1e9); } }); From c8172f581098af3b203e38777fd499fbc67e20f1 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 08:14:59 -0400 Subject: [PATCH 20/42] refactor(beacon-node): move initialization out of conditional --- packages/beacon-node/src/network/core/networkCoreWorker.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/src/network/core/networkCoreWorker.ts b/packages/beacon-node/src/network/core/networkCoreWorker.ts index 3fdddc175082..befdb83a4bfd 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorker.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorker.ts @@ -83,12 +83,11 @@ new AsyncIterableBridgeHandler(getReqRespBridgeReqEvents(reqRespBridgeEventBus), ); const reqRespBridgeRespCaller = new AsyncIterableBridgeCaller(getReqRespBridgeRespEvents(reqRespBridgeEventBus)); -let networkCoreWorkerMetrics: NetworkCoreWorkerMetrics | null = null; +const networkCoreWorkerMetrics = metricsRegister ? getNetworkCoreWorkerMetrics(metricsRegister) : null; // respBridgeCaller metrics -if (metricsRegister) { - networkCoreWorkerMetrics = getNetworkCoreWorkerMetrics(metricsRegister); +if (networkCoreWorkerMetrics) { networkCoreWorkerMetrics.reqRespBridgeRespCallerPending.addCollect(() => { - networkCoreWorkerMetrics?.reqRespBridgeRespCallerPending.set(reqRespBridgeRespCaller.pendingCount); + networkCoreWorkerMetrics.reqRespBridgeRespCallerPending.set(reqRespBridgeRespCaller.pendingCount); }); } From 0c84dcb770182905677f23118c5312d6de110af9 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 25 Jul 2023 08:25:29 -0400 Subject: [PATCH 21/42] fix: remove unused import --- packages/beacon-node/src/network/core/networkCoreWorker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/beacon-node/src/network/core/networkCoreWorker.ts b/packages/beacon-node/src/network/core/networkCoreWorker.ts index befdb83a4bfd..35303190a8f8 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorker.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorker.ts @@ -13,7 +13,7 @@ import {peerIdToString} from "../../util/peerId.js"; import {profileNodeJS} from "../../util/profile.js"; import {NetworkEventBus, NetworkEventData, networkEventDirection} from "../events.js"; import {wireEventsOnWorkerThread} from "../../util/workerEvents.js"; -import {NetworkCoreWorkerMetrics, getNetworkCoreWorkerMetrics} from "./metrics.js"; +import {getNetworkCoreWorkerMetrics} from "./metrics.js"; import {NetworkWorkerApi, NetworkWorkerData} from "./types.js"; import {NetworkCore} from "./networkCore.js"; import { From daa2640d8bf28b08387f89b5d15794b2da764c27 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 26 Jul 2023 21:54:00 -0400 Subject: [PATCH 22/42] feat(metrics): add unit to metric name --- packages/beacon-node/src/metrics/metrics/lodestar.ts | 2 +- packages/beacon-node/src/network/core/metrics.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index c8a86224ef40..d1eb7e9db81f 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -121,7 +121,7 @@ export function createLodestarMetrics( help: "Current count of pending items in reqRespBridgeReqCaller data structure", }), }, - networkWorkerWireEventsOnMainThreadLatency: register.histogram({ + networkWorkerWireEventsOnMainThreadLatencySec: register.histogram({ name: "lodestar_network_worker_wire_events_on_main_thread_latency", help: "Latency to transmit network events to main thread across worker port", buckets: [0.001, 0.003, 0.01, 0.03, 0.1], diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts index edfde7c61423..5017417f10bc 100644 --- a/packages/beacon-node/src/network/core/metrics.ts +++ b/packages/beacon-node/src/network/core/metrics.ts @@ -342,7 +342,7 @@ export function getNetworkCoreWorkerMetrics(register: RegistryMetricCreator) { name: "lodestar_network_worker_reqresp_bridge_caller_pending_count", help: "Current count of pending elements in respBridgeCaller", }), - networkWorkerWireEventsOnWorkerThreadLatency: register.histogram({ + networkWorkerWireEventsOnWorkerThreadLatencySec: register.histogram({ name: "lodestar_network_worker_wire_events_on_worker_thread_latency", help: "Latency to transmit network events to worker thread across parent port", buckets: [0.001, 0.003, 0.01, 0.03, 0.1], From 18e45619cf9bafdccb9fbf39a98103600e3af14a Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 26 Jul 2023 22:01:04 -0400 Subject: [PATCH 23/42] feat(metrics): add unit to metric name --- packages/beacon-node/src/util/workerEvents.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index ad18bba493e2..8ad259fdf389 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -43,7 +43,7 @@ export function wireEventsOnWorkerThread( ) { events.emit(data.event, data.data); const [sec, nanoSec] = process.hrtime(data.posted); - metrics?.networkWorkerWireEventsOnWorkerThreadLatency.observe(sec + nanoSec / 1e9); + metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe(sec + nanoSec / 1e9); } }); @@ -80,7 +80,7 @@ export function wireEventsOnMainThread( ) { events.emit(data.event, data.data); const [sec, nanoSec] = process.hrtime(data.posted); - metrics?.networkWorkerWireEventsOnMainThreadLatency.observe(sec + nanoSec / 1e9); + metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe(sec + nanoSec / 1e9); } }); From d18702c17983621a88f754b8af93adc7006e5da1 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 26 Jul 2023 22:28:27 -0400 Subject: [PATCH 24/42] feat: add trace log statement to network worker --- .../src/network/core/networkCoreWorker.ts | 2 ++ .../src/network/core/networkCoreWorkerHandler.ts | 2 ++ packages/beacon-node/src/util/workerEvents.ts | 15 +++++++++++---- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/src/network/core/networkCoreWorker.ts b/packages/beacon-node/src/network/core/networkCoreWorker.ts index 35303190a8f8..5ab9f157602f 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorker.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorker.ts @@ -111,6 +111,7 @@ wireEventsOnWorkerThread( events, parentPort, networkCoreWorkerMetrics, + logger, networkEventDirection ); wireEventsOnWorkerThread( @@ -118,6 +119,7 @@ wireEventsOnWorkerThread( reqRespBridgeEventBus, parentPort, networkCoreWorkerMetrics, + logger, reqRespBridgeEventDirection ); diff --git a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts index 8c944dd87d2d..1d283b888af6 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts @@ -76,6 +76,7 @@ export class WorkerNetworkCore implements INetworkCore { modules.events, modules.worker as unknown as worker_threads.Worker, modules.metrics, + modules.logger, networkEventDirection ); wireEventsOnMainThread( @@ -83,6 +84,7 @@ export class WorkerNetworkCore implements INetworkCore { this.reqRespBridgeEventBus, modules.worker as unknown as worker_threads.Worker, modules.metrics, + modules.logger, reqRespBridgeEventDirection ); diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 8ad259fdf389..4f6ff3695f88 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -2,6 +2,7 @@ import {MessagePort, Worker} from "node:worker_threads"; import {Thread} from "@chainsafe/threads"; import {Logger} from "@lodestar/logger"; import {sleep} from "@lodestar/utils"; +import {LoggerNode} from "@lodestar/logger/lib/node.js"; import {Metrics} from "../metrics/metrics.js"; import {NetworkCoreWorkerMetrics} from "../network/core/metrics.js"; import {StrictEventEmitterSingleArg} from "./strictEvents.js"; @@ -31,6 +32,7 @@ export function wireEventsOnWorkerThread( events: StrictEventEmitterSingleArg, parentPort: MessagePort, metrics: NetworkCoreWorkerMetrics | null, + logger: LoggerNode, isWorkerToMain: {[K in keyof EventData]: EventDirection} ): void { // Subscribe to events from main thread @@ -41,9 +43,11 @@ export function wireEventsOnWorkerThread( // This check is not necessary but added for safety in case of improper implemented events isWorkerToMain[data.event] === EventDirection.mainToWorker ) { - events.emit(data.event, data.data); const [sec, nanoSec] = process.hrtime(data.posted); - metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe(sec + nanoSec / 1e9); + const networkWorkerLatency = sec + nanoSec / 1e9; + metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe(networkWorkerLatency); + logger.trace("network worker message latency", networkWorkerLatency); + events.emit(data.event, data.data); } }); @@ -68,6 +72,7 @@ export function wireEventsOnMainThread( events: StrictEventEmitterSingleArg, worker: Pick, metrics: Metrics | null, + logger: LoggerNode, isWorkerToMain: {[K in keyof EventData]: EventDirection} ): void { // Subscribe to events from main thread @@ -78,9 +83,11 @@ export function wireEventsOnMainThread( // This check is not necessary but added for safety in case of improper implemented events isWorkerToMain[data.event] === EventDirection.workerToMain ) { - events.emit(data.event, data.data); const [sec, nanoSec] = process.hrtime(data.posted); - metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe(sec + nanoSec / 1e9); + const networkWorkerLatency = sec + nanoSec / 1e9; + metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe(networkWorkerLatency); + logger.trace("network worker message latency", networkWorkerLatency); + events.emit(data.event, data.data); } }); From 609956088eb7d60d799626defb160ea6d046b1b4 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 27 Jul 2023 00:25:35 -0400 Subject: [PATCH 25/42] fix: change trace to debug log --- packages/beacon-node/src/util/workerEvents.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 4f6ff3695f88..48cfff35add9 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -46,7 +46,7 @@ export function wireEventsOnWorkerThread( const [sec, nanoSec] = process.hrtime(data.posted); const networkWorkerLatency = sec + nanoSec / 1e9; metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe(networkWorkerLatency); - logger.trace("network worker message latency", networkWorkerLatency); + logger.debug("network worker message latency", networkWorkerLatency); events.emit(data.event, data.data); } }); @@ -86,7 +86,7 @@ export function wireEventsOnMainThread( const [sec, nanoSec] = process.hrtime(data.posted); const networkWorkerLatency = sec + nanoSec / 1e9; metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe(networkWorkerLatency); - logger.trace("network worker message latency", networkWorkerLatency); + logger.debug("network worker message latency", networkWorkerLatency); events.emit(data.event, data.data); } }); From 59492067a206e098bc89eb866e9e9d0700545ed9 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 27 Jul 2023 02:25:12 -0400 Subject: [PATCH 26/42] feat(metrics): add eventName to network worker message metrics --- .../beacon-node/src/metrics/metrics/lodestar.ts | 3 ++- packages/beacon-node/src/network/core/metrics.ts | 3 ++- packages/beacon-node/src/util/workerEvents.ts | 14 ++++++++++---- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index d1eb7e9db81f..eaad0a05b0e9 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -121,9 +121,10 @@ export function createLodestarMetrics( help: "Current count of pending items in reqRespBridgeReqCaller data structure", }), }, - networkWorkerWireEventsOnMainThreadLatencySec: register.histogram({ + networkWorkerWireEventsOnMainThreadLatencySec: register.histogram<"eventName">({ name: "lodestar_network_worker_wire_events_on_main_thread_latency", help: "Latency to transmit network events to main thread across worker port", + labelNames: ["eventName"], buckets: [0.001, 0.003, 0.01, 0.03, 0.1], }), diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts index 5017417f10bc..799d099b10d6 100644 --- a/packages/beacon-node/src/network/core/metrics.ts +++ b/packages/beacon-node/src/network/core/metrics.ts @@ -342,9 +342,10 @@ export function getNetworkCoreWorkerMetrics(register: RegistryMetricCreator) { name: "lodestar_network_worker_reqresp_bridge_caller_pending_count", help: "Current count of pending elements in respBridgeCaller", }), - networkWorkerWireEventsOnWorkerThreadLatencySec: register.histogram({ + networkWorkerWireEventsOnWorkerThreadLatencySec: register.histogram<"eventName">({ name: "lodestar_network_worker_wire_events_on_worker_thread_latency", help: "Latency to transmit network events to worker thread across parent port", + labelNames: ["eventName"], buckets: [0.001, 0.003, 0.01, 0.03, 0.1], }), }; diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 48cfff35add9..6ed4c2c09347 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -45,8 +45,11 @@ export function wireEventsOnWorkerThread( ) { const [sec, nanoSec] = process.hrtime(data.posted); const networkWorkerLatency = sec + nanoSec / 1e9; - metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe(networkWorkerLatency); - logger.debug("network worker message latency", networkWorkerLatency); + metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe( + {eventName: data.event as string}, + networkWorkerLatency + ); + logger.trace("network worker message latency", networkWorkerLatency); events.emit(data.event, data.data); } }); @@ -85,8 +88,11 @@ export function wireEventsOnMainThread( ) { const [sec, nanoSec] = process.hrtime(data.posted); const networkWorkerLatency = sec + nanoSec / 1e9; - metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe(networkWorkerLatency); - logger.debug("network worker message latency", networkWorkerLatency); + metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe( + {eventName: data.event as string}, + networkWorkerLatency + ); + logger.trace("network worker message latency", networkWorkerLatency); events.emit(data.event, data.data); } }); From e1ad935128ec25e261dc5eb81e4959a76fc7cbce Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 27 Jul 2023 03:30:02 -0400 Subject: [PATCH 27/42] bug(logger): check if trace is broken --- packages/beacon-node/src/util/workerEvents.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 6ed4c2c09347..c9fcfb5ffff9 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -49,7 +49,7 @@ export function wireEventsOnWorkerThread( {eventName: data.event as string}, networkWorkerLatency ); - logger.trace("network worker message latency", networkWorkerLatency); + // logger.trace("network worker message latency", networkWorkerLatency); events.emit(data.event, data.data); } }); @@ -92,7 +92,7 @@ export function wireEventsOnMainThread( {eventName: data.event as string}, networkWorkerLatency ); - logger.trace("network worker message latency", networkWorkerLatency); + // logger.trace("network worker message latency", networkWorkerLatency); events.emit(data.event, data.data); } }); From 106ae0738431dc98a053a27ae671e3b087504a53 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 27 Jul 2023 04:50:59 -0400 Subject: [PATCH 28/42] feat(metrics): add worker eventDirection label --- dashboards/lodestar_networking.json | 118 ++++++++++++++++-- .../src/metrics/metrics/lodestar.ts | 4 +- .../beacon-node/src/network/core/metrics.ts | 4 +- packages/beacon-node/src/util/workerEvents.ts | 10 +- 4 files changed, 123 insertions(+), 13 deletions(-) diff --git a/dashboards/lodestar_networking.json b/dashboards/lodestar_networking.json index 6e5d0fc8bfb3..df65d02b8707 100644 --- a/dashboards/lodestar_networking.json +++ b/dashboards/lodestar_networking.json @@ -928,8 +928,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -944,7 +943,7 @@ "gridPos": { "h": 6, "w": 12, - "x": 12, + "x": 0, "y": 2 }, "id": 148, @@ -969,9 +968,9 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency[6m])", + "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency_sum[6m])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_count[6m])", "interval": "", - "legendFormat": "to worker thread", + "legendFormat": "{{eventName}}", "refId": "to worker thread" }, { @@ -983,7 +982,112 @@ "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency[6m])", "hide": false, "interval": "", - "legendFormat": "to main thread", + "legendFormat": "{{eventName}}", + "refId": "to main thread" + } + ], + "title": "Network worker - to/from worker port latency", + "type": "timeseries" + }, + { + "description": "Async time to send a message across the worker/parent port", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 2 + }, + "id": 609, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency_sum[6m])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_count[6m])", + "interval": "", + "legendFormat": "{{eventDirection}}", + "refId": "to worker thread" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency_sum[6m])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_count[6m])", + "hide": false, + "interval": "", + "legendFormat": "{{eventDirection}}", "refId": "to main thread" } ], @@ -6693,6 +6797,6 @@ "timezone": "utc", "title": "Lodestar - networking", "uid": "lodestar_networking", - "version": 38, + "version": 43, "weekStart": "monday" } \ No newline at end of file diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index eaad0a05b0e9..b66ba2c07d73 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -121,10 +121,10 @@ export function createLodestarMetrics( help: "Current count of pending items in reqRespBridgeReqCaller data structure", }), }, - networkWorkerWireEventsOnMainThreadLatencySec: register.histogram<"eventName">({ + networkWorkerWireEventsOnMainThreadLatencySec: register.histogram<"eventName" | "eventDirection">({ name: "lodestar_network_worker_wire_events_on_main_thread_latency", help: "Latency to transmit network events to main thread across worker port", - labelNames: ["eventName"], + labelNames: ["eventName", "eventDirection"], buckets: [0.001, 0.003, 0.01, 0.03, 0.1], }), diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts index 799d099b10d6..0bc60d4e80bd 100644 --- a/packages/beacon-node/src/network/core/metrics.ts +++ b/packages/beacon-node/src/network/core/metrics.ts @@ -342,10 +342,10 @@ export function getNetworkCoreWorkerMetrics(register: RegistryMetricCreator) { name: "lodestar_network_worker_reqresp_bridge_caller_pending_count", help: "Current count of pending elements in respBridgeCaller", }), - networkWorkerWireEventsOnWorkerThreadLatencySec: register.histogram<"eventName">({ + networkWorkerWireEventsOnWorkerThreadLatencySec: register.histogram<"eventName" | "eventDirection">({ name: "lodestar_network_worker_wire_events_on_worker_thread_latency", help: "Latency to transmit network events to worker thread across parent port", - labelNames: ["eventName"], + labelNames: ["eventName", "eventDirection"], buckets: [0.001, 0.003, 0.01, 0.03, 0.1], }), }; diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index c9fcfb5ffff9..3a53574a9a3b 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -46,7 +46,10 @@ export function wireEventsOnWorkerThread( const [sec, nanoSec] = process.hrtime(data.posted); const networkWorkerLatency = sec + nanoSec / 1e9; metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe( - {eventName: data.event as string}, + { + eventName: data.event as string, + eventDirection: EventDirection.mainToWorker, + }, networkWorkerLatency ); // logger.trace("network worker message latency", networkWorkerLatency); @@ -89,7 +92,10 @@ export function wireEventsOnMainThread( const [sec, nanoSec] = process.hrtime(data.posted); const networkWorkerLatency = sec + nanoSec / 1e9; metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe( - {eventName: data.event as string}, + { + eventName: data.event as string, + eventDirection: EventDirection.workerToMain, + }, networkWorkerLatency ); // logger.trace("network worker message latency", networkWorkerLatency); From 570162b20078938409a8330101f2a4f3b1cf67b5 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 27 Jul 2023 04:59:13 -0400 Subject: [PATCH 29/42] fix(metrics): use string instead of enum for eventDirection --- packages/beacon-node/src/util/workerEvents.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 3a53574a9a3b..752b5215eda0 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -48,7 +48,7 @@ export function wireEventsOnWorkerThread( metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe( { eventName: data.event as string, - eventDirection: EventDirection.mainToWorker, + eventDirection: "mainToWorker", }, networkWorkerLatency ); @@ -94,7 +94,7 @@ export function wireEventsOnMainThread( metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe( { eventName: data.event as string, - eventDirection: EventDirection.workerToMain, + eventDirection: "workerToMain", }, networkWorkerLatency ); From 5f76c36e8db7167b75d09a9b5b5d02046bd6723c Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 27 Jul 2023 05:26:49 -0400 Subject: [PATCH 30/42] fix(metrics): remove eventDirection label --- packages/beacon-node/src/metrics/metrics/lodestar.ts | 4 ++-- packages/beacon-node/src/network/core/metrics.ts | 4 ++-- packages/beacon-node/src/util/workerEvents.ts | 10 ++-------- 3 files changed, 6 insertions(+), 12 deletions(-) diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index b66ba2c07d73..eaad0a05b0e9 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -121,10 +121,10 @@ export function createLodestarMetrics( help: "Current count of pending items in reqRespBridgeReqCaller data structure", }), }, - networkWorkerWireEventsOnMainThreadLatencySec: register.histogram<"eventName" | "eventDirection">({ + networkWorkerWireEventsOnMainThreadLatencySec: register.histogram<"eventName">({ name: "lodestar_network_worker_wire_events_on_main_thread_latency", help: "Latency to transmit network events to main thread across worker port", - labelNames: ["eventName", "eventDirection"], + labelNames: ["eventName"], buckets: [0.001, 0.003, 0.01, 0.03, 0.1], }), diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts index 0bc60d4e80bd..799d099b10d6 100644 --- a/packages/beacon-node/src/network/core/metrics.ts +++ b/packages/beacon-node/src/network/core/metrics.ts @@ -342,10 +342,10 @@ export function getNetworkCoreWorkerMetrics(register: RegistryMetricCreator) { name: "lodestar_network_worker_reqresp_bridge_caller_pending_count", help: "Current count of pending elements in respBridgeCaller", }), - networkWorkerWireEventsOnWorkerThreadLatencySec: register.histogram<"eventName" | "eventDirection">({ + networkWorkerWireEventsOnWorkerThreadLatencySec: register.histogram<"eventName">({ name: "lodestar_network_worker_wire_events_on_worker_thread_latency", help: "Latency to transmit network events to worker thread across parent port", - labelNames: ["eventName", "eventDirection"], + labelNames: ["eventName"], buckets: [0.001, 0.003, 0.01, 0.03, 0.1], }), }; diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 752b5215eda0..c9fcfb5ffff9 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -46,10 +46,7 @@ export function wireEventsOnWorkerThread( const [sec, nanoSec] = process.hrtime(data.posted); const networkWorkerLatency = sec + nanoSec / 1e9; metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe( - { - eventName: data.event as string, - eventDirection: "mainToWorker", - }, + {eventName: data.event as string}, networkWorkerLatency ); // logger.trace("network worker message latency", networkWorkerLatency); @@ -92,10 +89,7 @@ export function wireEventsOnMainThread( const [sec, nanoSec] = process.hrtime(data.posted); const networkWorkerLatency = sec + nanoSec / 1e9; metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe( - { - eventName: data.event as string, - eventDirection: "workerToMain", - }, + {eventName: data.event as string}, networkWorkerLatency ); // logger.trace("network worker message latency", networkWorkerLatency); From e76e6723582de5002d8084a2d53755f65bef6f35 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Mon, 31 Jul 2023 23:41:06 -0600 Subject: [PATCH 31/42] feat(dashboards): add average panel for network worker message --- dashboards/lodestar_networking.json | 117 ++++++++++++++++------------ 1 file changed, 69 insertions(+), 48 deletions(-) diff --git a/dashboards/lodestar_networking.json b/dashboards/lodestar_networking.json index df65d02b8707..963ee9c63d6b 100644 --- a/dashboards/lodestar_networking.json +++ b/dashboards/lodestar_networking.json @@ -990,50 +990,61 @@ "type": "timeseries" }, { + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 2 + }, + "id": 609, + "type": "timeseries", + "title": "Network Worker Message Latency - Worker to Main", + "pluginVersion": "7.4.5", "description": "Async time to send a message across the worker/parent port", "fieldConfig": { "defaults": { - "color": { - "mode": "palette-classic" - }, "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 23, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, "lineInterpolation": "linear", + "barAlignment": 0, "lineWidth": 1, + "fillOpacity": 23, + "gradientMode": "opacity", + "spanNulls": false, + "showPoints": "never", "pointSize": 5, + "stacking": { + "mode": "none", + "group": "A" + }, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", "scaleDistribution": { - "log": 2, - "type": "log" + "type": "log", + "log": 2 }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "axisCenteredZero": false, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false, + "graph": false }, "thresholdsStyle": { "mode": "off" } }, + "color": { + "mode": "palette-classic" + }, "mappings": [], "thresholds": { "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1045,27 +1056,19 @@ }, "overrides": [] }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 2 - }, - "id": 609, "options": { - "graph": {}, + "tooltip": { + "mode": "multi", + "sort": "none" + }, "legend": { - "calcs": [], + "showLegend": true, "displayMode": "list", "placement": "bottom", - "showLegend": true + "calcs": [] }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "graph": {} }, - "pluginVersion": "7.4.5", "targets": [ { "datasource": { @@ -1073,10 +1076,13 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency_sum[6m])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_count[6m])", + "expr": "avg(rate(lodestar_network_worker_wire_events_on_main_thread_latency_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_count[$rate_interval]))", + "hide": true, "interval": "", - "legendFormat": "{{eventDirection}}", - "refId": "to worker thread" + "legendFormat": "Average to Main", + "refId": "average", + "editorMode": "code", + "range": true }, { "datasource": { @@ -1084,15 +1090,30 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency_sum[6m])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_count[6m])", - "hide": false, + "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_count[$rate_interval])", "interval": "", - "legendFormat": "{{eventDirection}}", - "refId": "to main thread" + "legendFormat": "{{eventName}}", + "refId": "by eventName", + "editorMode": "code", + "range": true + }, + { + "datasource": { + "uid": "prometheus_local", + "type": "prometheus" + }, + "refId": "A", + "hide": false, + "editorMode": "code", + "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_count[$rate_interval])", + "legendFormat": "{{eventName}}", + "range": true } ], - "title": "Network worker - to/from worker port latency", - "type": "timeseries" + "datasource": { + "uid": "prometheus_local", + "type": "prometheus" + } }, { "datasource": { From f69bfbbd2f6754111208c6c183ed52062d0d11fa Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 1 Aug 2023 02:22:34 -0600 Subject: [PATCH 32/42] fix(metrics): update naming per Nico's suggestions --- dashboards/lodestar_networking.json | 10 +++++----- packages/beacon-node/src/metrics/metrics/lodestar.ts | 6 +++--- packages/beacon-node/src/network/core/metrics.ts | 6 +++--- packages/beacon-node/src/util/workerEvents.ts | 4 ++-- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/dashboards/lodestar_networking.json b/dashboards/lodestar_networking.json index 963ee9c63d6b..ee869c2b870f 100644 --- a/dashboards/lodestar_networking.json +++ b/dashboards/lodestar_networking.json @@ -968,7 +968,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency_sum[6m])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_count[6m])", + "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency_seconds_sum[6m])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_seconds_count[6m])", "interval": "", "legendFormat": "{{eventName}}", "refId": "to worker thread" @@ -979,7 +979,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency[6m])", + "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency_seconds[6m])", "hide": false, "interval": "", "legendFormat": "{{eventName}}", @@ -1076,7 +1076,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "avg(rate(lodestar_network_worker_wire_events_on_main_thread_latency_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_count[$rate_interval]))", + "expr": "avg(rate(lodestar_network_worker_wire_events_on_main_thread_latency_seconds_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_seconds_count[$rate_interval]))", "hide": true, "interval": "", "legendFormat": "Average to Main", @@ -1090,7 +1090,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_count[$rate_interval])", + "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency_seconds_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_seconds_count[$rate_interval])", "interval": "", "legendFormat": "{{eventName}}", "refId": "by eventName", @@ -1105,7 +1105,7 @@ "refId": "A", "hide": false, "editorMode": "code", - "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_count[$rate_interval])", + "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency_seconds_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_seconds_count[$rate_interval])", "legendFormat": "{{eventName}}", "range": true } diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index eaad0a05b0e9..5ffae34a9eeb 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -121,9 +121,9 @@ export function createLodestarMetrics( help: "Current count of pending items in reqRespBridgeReqCaller data structure", }), }, - networkWorkerWireEventsOnMainThreadLatencySec: register.histogram<"eventName">({ - name: "lodestar_network_worker_wire_events_on_main_thread_latency", - help: "Latency to transmit network events to main thread across worker port", + networkWorkerWireEventsOnMainThreadLatency: register.histogram<"eventName">({ + name: "lodestar_network_worker_wire_events_on_main_thread_latency_seconds", + help: "Latency in seconds to transmit network events to main thread across worker port", labelNames: ["eventName"], buckets: [0.001, 0.003, 0.01, 0.03, 0.1], }), diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts index 799d099b10d6..e5ce0bede447 100644 --- a/packages/beacon-node/src/network/core/metrics.ts +++ b/packages/beacon-node/src/network/core/metrics.ts @@ -342,9 +342,9 @@ export function getNetworkCoreWorkerMetrics(register: RegistryMetricCreator) { name: "lodestar_network_worker_reqresp_bridge_caller_pending_count", help: "Current count of pending elements in respBridgeCaller", }), - networkWorkerWireEventsOnWorkerThreadLatencySec: register.histogram<"eventName">({ - name: "lodestar_network_worker_wire_events_on_worker_thread_latency", - help: "Latency to transmit network events to worker thread across parent port", + networkWorkerWireEventsOnWorkerThreadLatency: register.histogram<"eventName">({ + name: "lodestar_network_worker_wire_events_on_worker_thread_latency_seconds", + help: "Latency in seconds to transmit network events to worker thread across parent port", labelNames: ["eventName"], buckets: [0.001, 0.003, 0.01, 0.03, 0.1], }), diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index c9fcfb5ffff9..6b3f2fee535c 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -45,7 +45,7 @@ export function wireEventsOnWorkerThread( ) { const [sec, nanoSec] = process.hrtime(data.posted); const networkWorkerLatency = sec + nanoSec / 1e9; - metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe( + metrics?.networkWorkerWireEventsOnWorkerThreadLatency.observe( {eventName: data.event as string}, networkWorkerLatency ); @@ -88,7 +88,7 @@ export function wireEventsOnMainThread( ) { const [sec, nanoSec] = process.hrtime(data.posted); const networkWorkerLatency = sec + nanoSec / 1e9; - metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe( + metrics?.networkWorkerWireEventsOnMainThreadLatency.observe( {eventName: data.event as string}, networkWorkerLatency ); From 1b92def6796dda3a4226b11ea6109ee1634ebfd7 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 1 Aug 2023 02:26:26 -0600 Subject: [PATCH 33/42] refactor: remove unused logger from workerEvents.ts --- packages/beacon-node/src/network/core/networkCoreWorker.ts | 2 -- .../beacon-node/src/network/core/networkCoreWorkerHandler.ts | 2 -- packages/beacon-node/src/util/workerEvents.ts | 5 ----- 3 files changed, 9 deletions(-) diff --git a/packages/beacon-node/src/network/core/networkCoreWorker.ts b/packages/beacon-node/src/network/core/networkCoreWorker.ts index 5ab9f157602f..35303190a8f8 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorker.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorker.ts @@ -111,7 +111,6 @@ wireEventsOnWorkerThread( events, parentPort, networkCoreWorkerMetrics, - logger, networkEventDirection ); wireEventsOnWorkerThread( @@ -119,7 +118,6 @@ wireEventsOnWorkerThread( reqRespBridgeEventBus, parentPort, networkCoreWorkerMetrics, - logger, reqRespBridgeEventDirection ); diff --git a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts index 1d283b888af6..8c944dd87d2d 100644 --- a/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts +++ b/packages/beacon-node/src/network/core/networkCoreWorkerHandler.ts @@ -76,7 +76,6 @@ export class WorkerNetworkCore implements INetworkCore { modules.events, modules.worker as unknown as worker_threads.Worker, modules.metrics, - modules.logger, networkEventDirection ); wireEventsOnMainThread( @@ -84,7 +83,6 @@ export class WorkerNetworkCore implements INetworkCore { this.reqRespBridgeEventBus, modules.worker as unknown as worker_threads.Worker, modules.metrics, - modules.logger, reqRespBridgeEventDirection ); diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 6b3f2fee535c..805bc0f2867c 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -2,7 +2,6 @@ import {MessagePort, Worker} from "node:worker_threads"; import {Thread} from "@chainsafe/threads"; import {Logger} from "@lodestar/logger"; import {sleep} from "@lodestar/utils"; -import {LoggerNode} from "@lodestar/logger/lib/node.js"; import {Metrics} from "../metrics/metrics.js"; import {NetworkCoreWorkerMetrics} from "../network/core/metrics.js"; import {StrictEventEmitterSingleArg} from "./strictEvents.js"; @@ -32,7 +31,6 @@ export function wireEventsOnWorkerThread( events: StrictEventEmitterSingleArg, parentPort: MessagePort, metrics: NetworkCoreWorkerMetrics | null, - logger: LoggerNode, isWorkerToMain: {[K in keyof EventData]: EventDirection} ): void { // Subscribe to events from main thread @@ -49,7 +47,6 @@ export function wireEventsOnWorkerThread( {eventName: data.event as string}, networkWorkerLatency ); - // logger.trace("network worker message latency", networkWorkerLatency); events.emit(data.event, data.data); } }); @@ -75,7 +72,6 @@ export function wireEventsOnMainThread( events: StrictEventEmitterSingleArg, worker: Pick, metrics: Metrics | null, - logger: LoggerNode, isWorkerToMain: {[K in keyof EventData]: EventDirection} ): void { // Subscribe to events from main thread @@ -92,7 +88,6 @@ export function wireEventsOnMainThread( {eventName: data.event as string}, networkWorkerLatency ); - // logger.trace("network worker message latency", networkWorkerLatency); events.emit(data.event, data.data); } }); From 6b04c02be864be6f284b618ae6ad65a1b521c73c Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Tue, 1 Aug 2023 02:37:33 -0600 Subject: [PATCH 34/42] fix(metrics): add network worker unit name back --- packages/beacon-node/src/metrics/metrics/lodestar.ts | 2 +- packages/beacon-node/src/network/core/metrics.ts | 2 +- packages/beacon-node/src/util/workerEvents.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 5ffae34a9eeb..b78097329afb 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -121,7 +121,7 @@ export function createLodestarMetrics( help: "Current count of pending items in reqRespBridgeReqCaller data structure", }), }, - networkWorkerWireEventsOnMainThreadLatency: register.histogram<"eventName">({ + networkWorkerWireEventsOnMainThreadLatencySec: register.histogram<"eventName">({ name: "lodestar_network_worker_wire_events_on_main_thread_latency_seconds", help: "Latency in seconds to transmit network events to main thread across worker port", labelNames: ["eventName"], diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts index e5ce0bede447..42e3b0b144d7 100644 --- a/packages/beacon-node/src/network/core/metrics.ts +++ b/packages/beacon-node/src/network/core/metrics.ts @@ -342,7 +342,7 @@ export function getNetworkCoreWorkerMetrics(register: RegistryMetricCreator) { name: "lodestar_network_worker_reqresp_bridge_caller_pending_count", help: "Current count of pending elements in respBridgeCaller", }), - networkWorkerWireEventsOnWorkerThreadLatency: register.histogram<"eventName">({ + networkWorkerWireEventsOnWorkerThreadLatencySec: register.histogram<"eventName">({ name: "lodestar_network_worker_wire_events_on_worker_thread_latency_seconds", help: "Latency in seconds to transmit network events to worker thread across parent port", labelNames: ["eventName"], diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 805bc0f2867c..54d52896be0d 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -43,7 +43,7 @@ export function wireEventsOnWorkerThread( ) { const [sec, nanoSec] = process.hrtime(data.posted); const networkWorkerLatency = sec + nanoSec / 1e9; - metrics?.networkWorkerWireEventsOnWorkerThreadLatency.observe( + metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe( {eventName: data.event as string}, networkWorkerLatency ); @@ -84,7 +84,7 @@ export function wireEventsOnMainThread( ) { const [sec, nanoSec] = process.hrtime(data.posted); const networkWorkerLatency = sec + nanoSec / 1e9; - metrics?.networkWorkerWireEventsOnMainThreadLatency.observe( + metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe( {eventName: data.event as string}, networkWorkerLatency ); From f7874c0e716e704e02a9358698b94086851a8557 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 2 Aug 2023 20:03:54 -0600 Subject: [PATCH 35/42] fix(beacon-node): use bigint for hrtime in worker message metric --- packages/beacon-node/src/util/workerEvents.ts | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 54d52896be0d..5b854daf8ed5 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -6,10 +6,12 @@ import {Metrics} from "../metrics/metrics.js"; import {NetworkCoreWorkerMetrics} from "../network/core/metrics.js"; import {StrictEventEmitterSingleArg} from "./strictEvents.js"; +const nanoToSecondConversion = 1e9; + export type WorkerBridgeEvent = { type: string; event: keyof EventData; - posted: [number, number]; + posted: bigint; data: EventData[keyof EventData]; }; @@ -41,11 +43,9 @@ export function wireEventsOnWorkerThread( // This check is not necessary but added for safety in case of improper implemented events isWorkerToMain[data.event] === EventDirection.mainToWorker ) { - const [sec, nanoSec] = process.hrtime(data.posted); - const networkWorkerLatency = sec + nanoSec / 1e9; metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe( {eventName: data.event as string}, - networkWorkerLatency + Number(process.hrtime.bigint() - data.posted) / nanoToSecondConversion ); events.emit(data.event, data.data); } @@ -58,7 +58,7 @@ export function wireEventsOnWorkerThread( const workerEvent: WorkerBridgeEvent = { type: mainEventName, event: eventName, - posted: process.hrtime(), + posted: process.hrtime.bigint(), data, }; parentPort.postMessage(workerEvent); @@ -82,11 +82,9 @@ export function wireEventsOnMainThread( // This check is not necessary but added for safety in case of improper implemented events isWorkerToMain[data.event] === EventDirection.workerToMain ) { - const [sec, nanoSec] = process.hrtime(data.posted); - const networkWorkerLatency = sec + nanoSec / 1e9; metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe( {eventName: data.event as string}, - networkWorkerLatency + Number(process.hrtime.bigint() - data.posted) / nanoToSecondConversion ); events.emit(data.event, data.data); } @@ -99,7 +97,7 @@ export function wireEventsOnMainThread( const workerEvent: WorkerBridgeEvent = { type: mainEventName, event: eventName, - posted: process.hrtime(), + posted: process.hrtime.bigint(), data, }; worker.postMessage(workerEvent); From 042b913f2b37734e47ce45cbe1152364d3f300e2 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 2 Aug 2023 20:10:41 -0600 Subject: [PATCH 36/42] fix(dashboards): remove dashboard changes. moved to PR#5827 --- dashboards/lodestar_networking.json | 1537 +++++++++------------------ 1 file changed, 530 insertions(+), 1007 deletions(-) diff --git a/dashboards/lodestar_networking.json b/dashboards/lodestar_networking.json index ee869c2b870f..7a18f218db7b 100644 --- a/dashboards/lodestar_networking.json +++ b/dashboards/lodestar_networking.json @@ -34,7 +34,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 1, - "id": 38, + "id": null, "links": [ { "asDropdown": true, @@ -54,7 +54,7 @@ "liveNow": false, "panels": [ { - "collapsed": true, + "collapsed": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" @@ -66,826 +66,552 @@ "y": 0 }, "id": 510, - "panels": [ + "panels": [], + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 20, - "gradientMode": "opacity", - "hideFrom": { - "graph": false, - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 4, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "normal" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 1 - }, - "id": 502, - "options": { - "graph": {}, - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "refId": "A" + } + ], + "title": "Summary", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "pluginVersion": "7.4.5", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "lodestar_peers_by_client_count", - "format": "time_series", - "interval": "", - "intervalFactor": 1, - "legendFormat": "{{client}}", - "refId": "A" + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 4, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" } - ], - "title": "Peers by client implementation", - "type": "timeseries" + }, + "mappings": [], + "min": 0, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 502, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "exemplar": false, + "expr": "lodestar_peers_by_client_count", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{client}}", + "refId": "A" + } + ], + "title": "Peers by client implementation", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 1 - }, - "id": 328, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "8.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "lodestar_gossip_mesh_peers_by_type_count{type!~\"beacon_attestation\"}", - "format": "time_series", - "interval": "", - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "title": "Mesh peers count", - "type": "timeseries" + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 328, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.2", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "exemplar": false, + "expr": "lodestar_gossip_mesh_peers_by_type_count{type!~\"beacon_attestation\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Mesh peers count", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, - "overrides": [] - }, - "gridPos": { - "h": 9, - "w": 12, - "x": 0, - "y": 9 - }, - "id": 330, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" + "thresholdsStyle": { + "mode": "off" } }, - "pluginVersion": "8.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "lodestar_gossip_peer_score_by_threshold_count", - "format": "time_series", - "interval": "", - "legendFormat": "{{threshold}}", - "refId": "A" - } - ], - "title": "Peer count per gossip score threshold", - "type": "timeseries" + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 330, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.2", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } + "exemplar": false, + "expr": "lodestar_gossip_peer_score_by_threshold_count", + "format": "time_series", + "interval": "", + "legendFormat": "{{threshold}}", + "refId": "A" + } + ], + "title": "Peer count per gossip score threshold", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "max" }, - "overrides": [ + "properties": [ { - "matcher": { - "id": "byName", - "options": "max" - }, - "properties": [ - { - "id": "custom.fillBelowTo", - "value": "avg" - }, - { - "id": "custom.fillOpacity", - "value": 12 - }, - { - "id": "color", - "value": { - "fixedColor": "semi-dark-green", - "mode": "fixed" - } - } - ] + "id": "custom.fillBelowTo", + "value": "avg" }, { - "matcher": { - "id": "byName", - "options": "avg" - }, - "properties": [ - { - "id": "custom.fillBelowTo", - "value": "min" - }, - { - "id": "custom.fillOpacity", - "value": 12 - }, - { - "id": "custom.gradientMode", - "value": "opacity" - }, - { - "id": "color", - "value": { - "fixedColor": "super-light-green", - "mode": "fixed" - } - } - ] + "id": "custom.fillOpacity", + "value": 12 }, { - "matcher": { - "id": "byName", - "options": "min" - }, - "properties": [ - { - "id": "color", - "value": { - "fixedColor": "dark-blue", - "mode": "fixed" - } - } - ] + "id": "color", + "value": { + "fixedColor": "semi-dark-green", + "mode": "fixed" + } } ] }, - "gridPos": { - "h": 9, - "w": 12, - "x": 12, - "y": 9 - }, - "id": 331, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "lodestar_gossip_score_avg_min_max_max", - "format": "time_series", - "interval": "", - "legendFormat": "max", - "refId": "max" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + { + "matcher": { + "id": "byName", + "options": "avg" + }, + "properties": [ + { + "id": "custom.fillBelowTo", + "value": "min" }, - "exemplar": false, - "expr": "lodestar_gossip_score_avg_min_max_avg", - "hide": false, - "interval": "", - "legendFormat": "avg", - "refId": "avg" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + { + "id": "custom.fillOpacity", + "value": 12 }, - "exemplar": false, - "expr": "lodestar_gossip_score_avg_min_max_min", - "hide": false, - "interval": "", - "legendFormat": "min", - "refId": "min" - } - ], - "title": "Gossip score avg / min / max", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" + { + "id": "custom.gradientMode", + "value": "opacity" }, - "decimals": 0, - "mappings": [], - "min": 0.01, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + { + "id": "color", + "value": { + "fixedColor": "super-light-green", + "mode": "fixed" + } } + ] + }, + { + "matcher": { + "id": "byName", + "options": "min" }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 0, - "y": 18 - }, - "id": 70, - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "text": {} - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "lodestar_gossip_mesh_peers_by_beacon_attestation_subnet_count", - "interval": "", - "legendFormat": "{{subnet}}", - "refId": "A" - } - ], - "title": "Attnet mesh peers per subnet", - "type": "bargauge" + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 331, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.2", + "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "thresholds" - }, - "mappings": [], - "min": 0.01, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 5, - "w": 12, - "x": 12, - "y": 18 - }, - "id": 68, - "options": { - "displayMode": "lcd", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "last" - ], - "fields": "", - "values": false - }, - "showUnfilled": true, - "text": {} - }, - "pluginVersion": "9.3.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "expr": "lodestar_gossip_mesh_peers_by_type_count{type!~\"beacon_attestation\"}", - "format": "time_series", - "interval": "", - "legendFormat": "{{type}}", - "refId": "A" - } - ], - "title": "Mesh peers", - "type": "bargauge" + "exemplar": false, + "expr": "lodestar_gossip_score_avg_min_max_max", + "format": "time_series", + "interval": "", + "legendFormat": "max", + "refId": "max" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 0, - "y": 23 - }, - "id": 300, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "8.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "count(lodestar_gossip_mesh_peers_by_beacon_attestation_subnet_count > 0)", - "interval": "", - "legendFormat": "{{subnet}}", - "refId": "A" - } - ], - "title": "Attnet count with > 0 mesh peers", - "type": "timeseries" + "exemplar": false, + "expr": "lodestar_gossip_score_avg_min_max_avg", + "hide": false, + "interval": "", + "legendFormat": "avg", + "refId": "avg" }, { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 7, - "w": 12, - "x": 12, - "y": 23 - }, - "id": 298, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "none" - } + "exemplar": false, + "expr": "lodestar_gossip_score_avg_min_max_min", + "hide": false, + "interval": "", + "legendFormat": "min", + "refId": "min" + } + ], + "title": "Gossip score avg / min / max", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" }, - "pluginVersion": "8.2.2", - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "lodestar_gossip_topic_peers_by_type_count{type!~\"beacon_attestation\"}", - "format": "time_series", - "interval": "", - "legendFormat": "{{type}}", - "refId": "A" - } + "decimals": 0, + "mappings": [], + "min": 0.01 + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 70, + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" ], - "title": "Topic peers count", - "type": "timeseries" - } - ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "9.3.2", "targets": [ { "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, + "exemplar": false, + "expr": "lodestar_gossip_mesh_peers_by_beacon_attestation_subnet_count", + "interval": "", + "legendFormat": "{{subnet}}", "refId": "A" } ], - "title": "Summary", - "type": "row" + "title": "Attnet mesh peers per subnet", + "type": "bargauge" }, { - "collapsed": false, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0.01 + }, + "overrides": [] + }, "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 1 + "h": 5, + "w": 12, + "x": 12, + "y": 18 }, - "id": 608, - "panels": [], - "title": "Network Worker", - "type": "row" + "id": 68, + "options": { + "displayMode": "lcd", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "text": {} + }, + "pluginVersion": "9.3.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "expr": "lodestar_gossip_mesh_peers_by_type_count{type!~\"beacon_attestation\"}", + "format": "time_series", + "interval": "", + "legendFormat": "{{type}}", + "refId": "A" + } + ], + "title": "Mesh peers", + "type": "bargauge" }, { - "description": "Async time to send a message across the worker/parent port", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, "fieldConfig": { "defaults": { "color": { @@ -898,10 +624,9 @@ "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 23, - "gradientMode": "opacity", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { - "graph": false, "legend": false, "tooltip": false, "viz": false @@ -910,10 +635,9 @@ "lineWidth": 1, "pointSize": 5, "scaleDistribution": { - "log": 2, - "type": "log" + "type": "linear" }, - "showPoints": "never", + "showPoints": "auto", "spanNulls": false, "stacking": { "group": "A", @@ -923,32 +647,18 @@ "mode": "off" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green" - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" + "mappings": [] }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 7, "w": 12, "x": 0, - "y": 2 + "y": 23 }, - "id": 148, + "id": 300, "options": { - "graph": {}, "legend": { "calcs": [], "displayMode": "list", @@ -956,11 +666,11 @@ "showLegend": true }, "tooltip": { - "mode": "multi", + "mode": "single", "sort": "none" } }, - "pluginVersion": "7.4.5", + "pluginVersion": "8.2.2", "targets": [ { "datasource": { @@ -968,107 +678,79 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency_seconds_sum[6m])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_seconds_count[6m])", - "interval": "", - "legendFormat": "{{eventName}}", - "refId": "to worker thread" - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency_seconds[6m])", - "hide": false, + "expr": "count(lodestar_gossip_mesh_peers_by_beacon_attestation_subnet_count > 0)", "interval": "", - "legendFormat": "{{eventName}}", - "refId": "to main thread" + "legendFormat": "{{subnet}}", + "refId": "A" } ], - "title": "Network worker - to/from worker port latency", + "title": "Attnet count with > 0 mesh peers", "type": "timeseries" }, { - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 2 + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" }, - "id": 609, - "type": "timeseries", - "title": "Network Worker Message Latency - Worker to Main", - "pluginVersion": "7.4.5", - "description": "Async time to send a message across the worker/parent port", "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, "lineInterpolation": "linear", - "barAlignment": 0, "lineWidth": 1, - "fillOpacity": 23, - "gradientMode": "opacity", - "spanNulls": false, - "showPoints": "never", "pointSize": 5, - "stacking": { - "mode": "none", - "group": "A" - }, - "axisPlacement": "auto", - "axisLabel": "", - "axisColorMode": "text", "scaleDistribution": { - "type": "log", - "log": 2 + "type": "linear" }, - "axisCenteredZero": false, - "hideFrom": { - "tooltip": false, - "viz": false, - "legend": false, - "graph": false + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" }, "thresholdsStyle": { "mode": "off" } }, - "color": { - "mode": "palette-classic" - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "s" + "mappings": [] }, "overrides": [] }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 23 + }, + "id": 298, "options": { - "tooltip": { - "mode": "multi", - "sort": "none" - }, "legend": { - "showLegend": true, + "calcs": [], "displayMode": "list", "placement": "bottom", - "calcs": [] + "showLegend": true }, - "graph": {} + "tooltip": { + "mode": "multi", + "sort": "none" + } }, + "pluginVersion": "8.2.2", "targets": [ { "datasource": { @@ -1076,44 +758,15 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "avg(rate(lodestar_network_worker_wire_events_on_main_thread_latency_seconds_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_seconds_count[$rate_interval]))", - "hide": true, - "interval": "", - "legendFormat": "Average to Main", - "refId": "average", - "editorMode": "code", - "range": true - }, - { - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" - }, - "exemplar": false, - "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency_seconds_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_seconds_count[$rate_interval])", + "expr": "lodestar_gossip_topic_peers_by_type_count{type!~\"beacon_attestation\"}", + "format": "time_series", "interval": "", - "legendFormat": "{{eventName}}", - "refId": "by eventName", - "editorMode": "code", - "range": true - }, - { - "datasource": { - "uid": "prometheus_local", - "type": "prometheus" - }, - "refId": "A", - "hide": false, - "editorMode": "code", - "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency_seconds_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_seconds_count[$rate_interval])", - "legendFormat": "{{eventName}}", - "range": true + "legendFormat": "{{type}}", + "refId": "A" } ], - "datasource": { - "uid": "prometheus_local", - "type": "prometheus" - } + "title": "Topic peers count", + "type": "timeseries" }, { "datasource": { @@ -1503,19 +1156,6 @@ }, "mappings": [], "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1601,19 +1241,6 @@ }, "mappings": [], "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1702,19 +1329,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -1912,19 +1526,6 @@ }, "mappings": [], "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -2234,19 +1835,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [ @@ -2444,19 +2032,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "short" }, "overrides": [] @@ -3078,19 +2653,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -3174,19 +2736,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "none" }, "overrides": [] @@ -3270,19 +2819,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "s" }, "overrides": [] @@ -3367,19 +2903,6 @@ } }, "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, "unit": "percentunit" }, "overrides": [] @@ -6787,7 +6310,7 @@ "condition": "", "key": "instance", "operator": "=", - "value": "feat1-lg1k-hzax41" + "value": "unstable-lg1k-hzax41" } ], "hide": 0, @@ -6818,6 +6341,6 @@ "timezone": "utc", "title": "Lodestar - networking", "uid": "lodestar_networking", - "version": 43, + "version": 14, "weekStart": "monday" -} \ No newline at end of file +} From 8477afb8384af564eccf2f183cdf2cc02b9e755c Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 2 Aug 2023 20:12:19 -0600 Subject: [PATCH 37/42] fix(dashboards): remove dashboard changes. moved to PR#5827 --- dashboards/lodestar_networking.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dashboards/lodestar_networking.json b/dashboards/lodestar_networking.json index 7a18f218db7b..65f1ea246a98 100644 --- a/dashboards/lodestar_networking.json +++ b/dashboards/lodestar_networking.json @@ -6343,4 +6343,5 @@ "uid": "lodestar_networking", "version": 14, "weekStart": "monday" -} + +} \ No newline at end of file From e106735165c85418d0f86dd68e59a81603b96791 Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Wed, 2 Aug 2023 20:14:39 -0600 Subject: [PATCH 38/42] fix(dashboards): remove dashboard changes. moved to PR#5827 --- dashboards/lodestar_networking.json | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/dashboards/lodestar_networking.json b/dashboards/lodestar_networking.json index 65f1ea246a98..e56365698e82 100644 --- a/dashboards/lodestar_networking.json +++ b/dashboards/lodestar_networking.json @@ -1,12 +1,12 @@ { "__inputs": [ { - "description": "", - "label": "Prometheus", "name": "DS_PROMETHEUS", + "type": "datasource", + "label": "Prometheus", + "description": "", "pluginId": "prometheus", - "pluginName": "Prometheus", - "type": "datasource" + "pluginName": "Prometheus" } ], "annotations": { @@ -6343,5 +6343,4 @@ "uid": "lodestar_networking", "version": 14, "weekStart": "monday" - -} \ No newline at end of file +} From cfcb7a61a3c90fe1cc8791f12e442b9c20cd7bc4 Mon Sep 17 00:00:00 2001 From: Matthew Keil Date: Thu, 12 Oct 2023 02:05:47 +0200 Subject: [PATCH 39/42] fix: constant case for nano conversion --- packages/beacon-node/src/util/workerEvents.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 5b854daf8ed5..579d844e33fd 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -6,7 +6,7 @@ import {Metrics} from "../metrics/metrics.js"; import {NetworkCoreWorkerMetrics} from "../network/core/metrics.js"; import {StrictEventEmitterSingleArg} from "./strictEvents.js"; -const nanoToSecondConversion = 1e9; +const NANO_TO_SECOND_CONVERSION = 1e9; export type WorkerBridgeEvent = { type: string; @@ -45,7 +45,7 @@ export function wireEventsOnWorkerThread( ) { metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe( {eventName: data.event as string}, - Number(process.hrtime.bigint() - data.posted) / nanoToSecondConversion + Number(process.hrtime.bigint() - data.posted) / NANO_TO_SECOND_CONVERSION ); events.emit(data.event, data.data); } @@ -84,7 +84,7 @@ export function wireEventsOnMainThread( ) { metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe( {eventName: data.event as string}, - Number(process.hrtime.bigint() - data.posted) / nanoToSecondConversion + Number(process.hrtime.bigint() - data.posted) / NANO_TO_SECOND_CONVERSION ); events.emit(data.event, data.data); } From e80243789f03a1aad31d63b02698a5710c414a83 Mon Sep 17 00:00:00 2001 From: Matthew Keil Date: Thu, 12 Oct 2023 02:10:16 +0200 Subject: [PATCH 40/42] Revert "fix(beacon-node): use bigint for hrtime in worker message metric" This reverts commit 59f12ff89cdc1cc98c33befc1e980d777419728a. --- packages/beacon-node/src/util/workerEvents.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 579d844e33fd..4d203e4a0c13 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -11,7 +11,7 @@ const NANO_TO_SECOND_CONVERSION = 1e9; export type WorkerBridgeEvent = { type: string; event: keyof EventData; - posted: bigint; + posted: [number, number]; data: EventData[keyof EventData]; }; @@ -43,9 +43,11 @@ export function wireEventsOnWorkerThread( // This check is not necessary but added for safety in case of improper implemented events isWorkerToMain[data.event] === EventDirection.mainToWorker ) { + const [sec, nanoSec] = process.hrtime(data.posted); + const networkWorkerLatency = sec + nanoSec / NANO_TO_SECOND_CONVERSION; metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe( {eventName: data.event as string}, - Number(process.hrtime.bigint() - data.posted) / NANO_TO_SECOND_CONVERSION + networkWorkerLatency ); events.emit(data.event, data.data); } @@ -58,7 +60,7 @@ export function wireEventsOnWorkerThread( const workerEvent: WorkerBridgeEvent = { type: mainEventName, event: eventName, - posted: process.hrtime.bigint(), + posted: process.hrtime(), data, }; parentPort.postMessage(workerEvent); @@ -82,9 +84,11 @@ export function wireEventsOnMainThread( // This check is not necessary but added for safety in case of improper implemented events isWorkerToMain[data.event] === EventDirection.workerToMain ) { + const [sec, nanoSec] = process.hrtime(data.posted); + const networkWorkerLatency = sec + nanoSec / NANO_TO_SECOND_CONVERSION; metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe( {eventName: data.event as string}, - Number(process.hrtime.bigint() - data.posted) / NANO_TO_SECOND_CONVERSION + networkWorkerLatency ); events.emit(data.event, data.data); } @@ -97,7 +101,7 @@ export function wireEventsOnMainThread( const workerEvent: WorkerBridgeEvent = { type: mainEventName, event: eventName, - posted: process.hrtime.bigint(), + posted: process.hrtime(), data, }; worker.postMessage(workerEvent); From 537d465612229644430adf959693c8438015376d Mon Sep 17 00:00:00 2001 From: Matthew Keil Date: Thu, 12 Oct 2023 02:22:23 +0200 Subject: [PATCH 41/42] refactor: remove Sec suffix in metric variable name --- packages/beacon-node/src/metrics/metrics/lodestar.ts | 2 +- packages/beacon-node/src/network/core/metrics.ts | 2 +- packages/beacon-node/src/util/workerEvents.ts | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index b78097329afb..5ffae34a9eeb 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -121,7 +121,7 @@ export function createLodestarMetrics( help: "Current count of pending items in reqRespBridgeReqCaller data structure", }), }, - networkWorkerWireEventsOnMainThreadLatencySec: register.histogram<"eventName">({ + networkWorkerWireEventsOnMainThreadLatency: register.histogram<"eventName">({ name: "lodestar_network_worker_wire_events_on_main_thread_latency_seconds", help: "Latency in seconds to transmit network events to main thread across worker port", labelNames: ["eventName"], diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts index 42e3b0b144d7..e5ce0bede447 100644 --- a/packages/beacon-node/src/network/core/metrics.ts +++ b/packages/beacon-node/src/network/core/metrics.ts @@ -342,7 +342,7 @@ export function getNetworkCoreWorkerMetrics(register: RegistryMetricCreator) { name: "lodestar_network_worker_reqresp_bridge_caller_pending_count", help: "Current count of pending elements in respBridgeCaller", }), - networkWorkerWireEventsOnWorkerThreadLatencySec: register.histogram<"eventName">({ + networkWorkerWireEventsOnWorkerThreadLatency: register.histogram<"eventName">({ name: "lodestar_network_worker_wire_events_on_worker_thread_latency_seconds", help: "Latency in seconds to transmit network events to worker thread across parent port", labelNames: ["eventName"], diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index 4d203e4a0c13..807bf7a30618 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -45,7 +45,7 @@ export function wireEventsOnWorkerThread( ) { const [sec, nanoSec] = process.hrtime(data.posted); const networkWorkerLatency = sec + nanoSec / NANO_TO_SECOND_CONVERSION; - metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe( + metrics?.networkWorkerWireEventsOnWorkerThreadLatency.observe( {eventName: data.event as string}, networkWorkerLatency ); @@ -86,7 +86,7 @@ export function wireEventsOnMainThread( ) { const [sec, nanoSec] = process.hrtime(data.posted); const networkWorkerLatency = sec + nanoSec / NANO_TO_SECOND_CONVERSION; - metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe( + metrics?.networkWorkerWireEventsOnMainThreadLatency.observe( {eventName: data.event as string}, networkWorkerLatency ); From f9ed6bb50d09f42cfcc9b858780965804bac5e9e Mon Sep 17 00:00:00 2001 From: Matthew Keil Date: Thu, 12 Oct 2023 02:49:20 +0200 Subject: [PATCH 42/42] fix(dashboard): make metric name match the updates in PR --- dashboards/lodestar_networking.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dashboards/lodestar_networking.json b/dashboards/lodestar_networking.json index e56365698e82..8633faeb7668 100644 --- a/dashboards/lodestar_networking.json +++ b/dashboards/lodestar_networking.json @@ -937,7 +937,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "(\n sum(rate(\n lodestar_network_worker_wire_events_on_worker_thread_latency_sum[$rate_interval]\n )) \n +\n sum(rate(\n lodestar_network_worker_wire_events_on_main_thread_latency_sum[$rate_interval]\n ))\n)\n/\n(\n sum(rate(\n lodestar_network_worker_wire_events_on_worker_thread_latency_count[$rate_interval]\n ))\n +\n sum(rate(\n lodestar_network_worker_wire_events_on_main_thread_latency_count[$rate_interval]\n ))\n)", + "expr": "(\n sum(rate(\n lodestar_network_worker_wire_events_on_worker_thread_latency_seconds_sum[$rate_interval]\n )) \n +\n sum(rate(\n lodestar_network_worker_wire_events_on_main_thread_latency_seconds_sum[$rate_interval]\n ))\n)\n/\n(\n sum(rate(\n lodestar_network_worker_wire_events_on_worker_thread_latency_seconds_count[$rate_interval]\n ))\n +\n sum(rate(\n lodestar_network_worker_wire_events_on_main_thread_latency_seconds_count[$rate_interval]\n ))\n)", "hide": false, "interval": "", "legendFormat": "Average", @@ -951,7 +951,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg(rate(lodestar_network_worker_wire_events_on_worker_thread_latency_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_count[$rate_interval]))", + "expr": "avg(rate(lodestar_network_worker_wire_events_on_worker_thread_latency_seconds_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_seconds_count[$rate_interval]))", "hide": false, "interval": "", "legendFormat": "Worker to Main", @@ -964,7 +964,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "avg(rate(lodestar_network_worker_wire_events_on_main_thread_latency_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_count[$rate_interval]))", + "expr": "avg(rate(lodestar_network_worker_wire_events_on_main_thread_latency_seconds_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_seconds_count[$rate_interval]))", "hide": false, "legendFormat": "Main to Worker", "range": true, @@ -1051,7 +1051,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "avg(rate(lodestar_network_worker_wire_events_on_main_thread_latency_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_count[$rate_interval]))", + "expr": "avg(rate(lodestar_network_worker_wire_events_on_main_thread_latency_seconds_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_seconds_count[$rate_interval]))", "hide": true, "interval": "", "legendFormat": "Average to Main", @@ -1065,7 +1065,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_count[$rate_interval])", + "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency_seconds_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_seconds_count[$rate_interval])", "interval": "", "legendFormat": "{{eventName}}", "range": true, @@ -1077,7 +1077,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_count[$rate_interval])", + "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency_seconds_sum[$rate_interval])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_seconds_count[$rate_interval])", "hide": false, "legendFormat": "{{eventName}}", "range": true,