From 68b6b0bcf4c5e1b70ae02a6fcfa5c521cc7feaca Mon Sep 17 00:00:00 2001 From: matthewkeil Date: Thu, 27 Jul 2023 04:50:59 -0400 Subject: [PATCH] feat(metrics): add worker eventDirection label --- dashboards/lodestar_networking.json | 118 ++++++++++++++++-- .../src/metrics/metrics/lodestar.ts | 4 +- .../beacon-node/src/network/core/metrics.ts | 4 +- packages/beacon-node/src/util/workerEvents.ts | 10 +- 4 files changed, 123 insertions(+), 13 deletions(-) diff --git a/dashboards/lodestar_networking.json b/dashboards/lodestar_networking.json index bfe70abc649a..b6ae0408d623 100644 --- a/dashboards/lodestar_networking.json +++ b/dashboards/lodestar_networking.json @@ -918,8 +918,7 @@ "mode": "absolute", "steps": [ { - "color": "green", - "value": null + "color": "green" }, { "color": "red", @@ -934,7 +933,7 @@ "gridPos": { "h": 6, "w": 12, - "x": 12, + "x": 0, "y": 2 }, "id": 148, @@ -959,9 +958,9 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": false, - "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency[6m])", + "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency_sum[6m])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_count[6m])", "interval": "", - "legendFormat": "to worker thread", + "legendFormat": "{{eventName}}", "refId": "to worker thread" }, { @@ -973,7 +972,112 @@ "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency[6m])", "hide": false, "interval": "", - "legendFormat": "to main thread", + "legendFormat": "{{eventName}}", + "refId": "to main thread" + } + ], + "title": "Network worker - to/from worker port latency", + "type": "timeseries" + }, + { + "description": "Async time to send a message across the worker/parent port", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 2, + "type": "log" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 2 + }, + "id": 609, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency_sum[6m])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_count[6m])", + "interval": "", + "legendFormat": "{{eventDirection}}", + "refId": "to worker thread" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": false, + "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency_sum[6m])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_count[6m])", + "hide": false, + "interval": "", + "legendFormat": "{{eventDirection}}", "refId": "to main thread" } ], @@ -5834,6 +5938,6 @@ "timezone": "utc", "title": "Lodestar - networking", "uid": "lodestar_networking", - "version": 38, + "version": 43, "weekStart": "monday" } \ No newline at end of file diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts index 8b55faa83c55..130ea290784c 100644 --- a/packages/beacon-node/src/metrics/metrics/lodestar.ts +++ b/packages/beacon-node/src/metrics/metrics/lodestar.ts @@ -110,10 +110,10 @@ export function createLodestarMetrics( help: "Current count of pending items in reqRespBridgeReqCaller data structure", }), }, - networkWorkerWireEventsOnMainThreadLatencySec: register.histogram<"eventName">({ + networkWorkerWireEventsOnMainThreadLatencySec: register.histogram<"eventName" | "eventDirection">({ name: "lodestar_network_worker_wire_events_on_main_thread_latency", help: "Latency to transmit network events to main thread across worker port", - labelNames: ["eventName"], + labelNames: ["eventName", "eventDirection"], buckets: [0.001, 0.003, 0.01, 0.03, 0.1], }), diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts index 7bddeafbe5c3..9e7c8d7f5137 100644 --- a/packages/beacon-node/src/network/core/metrics.ts +++ b/packages/beacon-node/src/network/core/metrics.ts @@ -318,10 +318,10 @@ export function getNetworkCoreWorkerMetrics(register: RegistryMetricCreator) { name: "lodestar_network_worker_reqresp_bridge_caller_pending_count", help: "Current count of pending elements in respBridgeCaller", }), - networkWorkerWireEventsOnWorkerThreadLatencySec: register.histogram<"eventName">({ + networkWorkerWireEventsOnWorkerThreadLatencySec: register.histogram<"eventName" | "eventDirection">({ name: "lodestar_network_worker_wire_events_on_worker_thread_latency", help: "Latency to transmit network events to worker thread across parent port", - labelNames: ["eventName"], + labelNames: ["eventName", "eventDirection"], buckets: [0.001, 0.003, 0.01, 0.03, 0.1], }), }; diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts index a839a1c0b61c..124f528d164f 100644 --- a/packages/beacon-node/src/util/workerEvents.ts +++ b/packages/beacon-node/src/util/workerEvents.ts @@ -43,7 +43,10 @@ export function wireEventsOnWorkerThread( const [sec, nanoSec] = process.hrtime(data.posted); const networkWorkerLatency = sec + nanoSec / 1e9; metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe( - {eventName: data.event as string}, + { + eventName: data.event as string, + eventDirection: EventDirection.mainToWorker, + }, networkWorkerLatency ); // logger.trace("network worker message latency", networkWorkerLatency); @@ -86,7 +89,10 @@ export function wireEventsOnMainThread( const [sec, nanoSec] = process.hrtime(data.posted); const networkWorkerLatency = sec + nanoSec / 1e9; metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe( - {eventName: data.event as string}, + { + eventName: data.event as string, + eventDirection: EventDirection.workerToMain, + }, networkWorkerLatency ); // logger.trace("network worker message latency", networkWorkerLatency);