feat(metrics): add worker eventDirection label

ChainSafe · Jul 27, 2023 · 68b6b0b · 68b6b0b
1 parent 2c3d7e1
commit 68b6b0b
Show file tree

Hide file tree

Showing 4 changed files with 123 additions and 13 deletions.
diff --git a/dashboards/lodestar_networking.json b/dashboards/lodestar_networking.json
@@ -918,8 +918,7 @@
             "mode": "absolute",
             "steps": [
               {
-                "color": "green",
-                "value": null
+                "color": "green"
               },
               {
                 "color": "red",
@@ -934,7 +933,7 @@
       "gridPos": {
         "h": 6,
         "w": 12,
-        "x": 12,
+        "x": 0,
         "y": 2
       },
       "id": 148,
@@ -959,9 +958,9 @@
             "uid": "${DS_PROMETHEUS}"
           },
           "exemplar": false,
-          "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency[6m])",
+          "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency_sum[6m])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_count[6m])",
           "interval": "",
-          "legendFormat": "to worker thread",
+          "legendFormat": "{{eventName}}",
           "refId": "to worker thread"
         },
         {
@@ -973,7 +972,112 @@
           "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency[6m])",
           "hide": false,
           "interval": "",
-          "legendFormat": "to main thread",
+          "legendFormat": "{{eventName}}",
+          "refId": "to main thread"
+        }
+      ],
+      "title": "Network worker - to/from worker port latency",
+      "type": "timeseries"
+    },
+    {
+      "description": "Async time to send a message across the worker/parent port",
+      "fieldConfig": {
+        "defaults": {
+          "color": {
+            "mode": "palette-classic"
+          },
+          "custom": {
+            "axisCenteredZero": false,
+            "axisColorMode": "text",
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "line",
+            "fillOpacity": 23,
+            "gradientMode": "opacity",
+            "hideFrom": {
+              "graph": false,
+              "legend": false,
+              "tooltip": false,
+              "viz": false
+            },
+            "lineInterpolation": "linear",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": {
+              "log": 2,
+              "type": "log"
+            },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": {
+              "group": "A",
+              "mode": "none"
+            },
+            "thresholdsStyle": {
+              "mode": "off"
+            }
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green"
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          },
+          "unit": "s"
+        },
+        "overrides": []
+      },
+      "gridPos": {
+        "h": 6,
+        "w": 12,
+        "x": 12,
+        "y": 2
+      },
+      "id": 609,
+      "options": {
+        "graph": {},
+        "legend": {
+          "calcs": [],
+          "displayMode": "list",
+          "placement": "bottom",
+          "showLegend": true
+        },
+        "tooltip": {
+          "mode": "multi",
+          "sort": "none"
+        }
+      },
+      "pluginVersion": "7.4.5",
+      "targets": [
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "exemplar": false,
+          "expr": "rate(lodestar_network_worker_wire_events_on_worker_thread_latency_sum[6m])/rate(lodestar_network_worker_wire_events_on_worker_thread_latency_count[6m])",
+          "interval": "",
+          "legendFormat": "{{eventDirection}}",
+          "refId": "to worker thread"
+        },
+        {
+          "datasource": {
+            "type": "prometheus",
+            "uid": "${DS_PROMETHEUS}"
+          },
+          "exemplar": false,
+          "expr": "rate(lodestar_network_worker_wire_events_on_main_thread_latency_sum[6m])/rate(lodestar_network_worker_wire_events_on_main_thread_latency_count[6m])",
+          "hide": false,
+          "interval": "",
+          "legendFormat": "{{eventDirection}}",
           "refId": "to main thread"
         }
       ],
@@ -5834,6 +5938,6 @@
   "timezone": "utc",
   "title": "Lodestar - networking",
   "uid": "lodestar_networking",
-  "version": 38,
+  "version": 43,
   "weekStart": "monday"
 }
diff --git a/packages/beacon-node/src/metrics/metrics/lodestar.ts b/packages/beacon-node/src/metrics/metrics/lodestar.ts
@@ -110,10 +110,10 @@ export function createLodestarMetrics(
         help: "Current count of pending items in reqRespBridgeReqCaller data structure",
       }),
     },
-    networkWorkerWireEventsOnMainThreadLatencySec: register.histogram<"eventName">({
+    networkWorkerWireEventsOnMainThreadLatencySec: register.histogram<"eventName" | "eventDirection">({
       name: "lodestar_network_worker_wire_events_on_main_thread_latency",
       help: "Latency to transmit network events to main thread across worker port",
-      labelNames: ["eventName"],
+      labelNames: ["eventName", "eventDirection"],
       buckets: [0.001, 0.003, 0.01, 0.03, 0.1],
     }),
 

diff --git a/packages/beacon-node/src/network/core/metrics.ts b/packages/beacon-node/src/network/core/metrics.ts
@@ -318,10 +318,10 @@ export function getNetworkCoreWorkerMetrics(register: RegistryMetricCreator) {
       name: "lodestar_network_worker_reqresp_bridge_caller_pending_count",
       help: "Current count of pending elements in respBridgeCaller",
     }),
-    networkWorkerWireEventsOnWorkerThreadLatencySec: register.histogram<"eventName">({
+    networkWorkerWireEventsOnWorkerThreadLatencySec: register.histogram<"eventName" | "eventDirection">({
       name: "lodestar_network_worker_wire_events_on_worker_thread_latency",
       help: "Latency to transmit network events to worker thread across parent port",
-      labelNames: ["eventName"],
+      labelNames: ["eventName", "eventDirection"],
       buckets: [0.001, 0.003, 0.01, 0.03, 0.1],
     }),
   };

diff --git a/packages/beacon-node/src/util/workerEvents.ts b/packages/beacon-node/src/util/workerEvents.ts
@@ -43,7 +43,10 @@ export function wireEventsOnWorkerThread<EventData>(
       const [sec, nanoSec] = process.hrtime(data.posted);
       const networkWorkerLatency = sec + nanoSec / 1e9;
       metrics?.networkWorkerWireEventsOnWorkerThreadLatencySec.observe(
-        {eventName: data.event as string},
+        {
+          eventName: data.event as string,
+          eventDirection: EventDirection.mainToWorker,
+        },
         networkWorkerLatency
       );
       // logger.trace("network worker message latency", networkWorkerLatency);
@@ -86,7 +89,10 @@ export function wireEventsOnMainThread<EventData>(
       const [sec, nanoSec] = process.hrtime(data.posted);
       const networkWorkerLatency = sec + nanoSec / 1e9;
       metrics?.networkWorkerWireEventsOnMainThreadLatencySec.observe(
-        {eventName: data.event as string},
+        {
+          eventName: data.event as string,
+          eventDirection: EventDirection.workerToMain,
+        },
         networkWorkerLatency
       );
       // logger.trace("network worker message latency", networkWorkerLatency);