diff --git a/dashboards/lodestar_vm_host.json b/dashboards/lodestar_vm_host.json index 7471defd0e8e..457d4d120fa4 100644 --- a/dashboards/lodestar_vm_host.json +++ b/dashboards/lodestar_vm_host.json @@ -177,7 +177,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "(sum(nodejs_heap_size_total_bytes) + sum(discv5_worker_nodejs_heap_size_total_bytes) + sum(network_worker_nodejs_heap_size_total_bytes))\nor\n(sum(nodejs_heap_size_total_bytes) + sum(discv5_worker_nodejs_heap_size_total_bytes))", + "expr": "(sum(nodejs_heap_size_total_bytes) + sum(discv5_worker_nodejs_heap_size_total_bytes) + sum(network_worker_nodejs_heap_size_total_bytes) + sum(lodestar_historical_state_worker_nodejs_heap_size_total_bytes))\nor\n(sum(nodejs_heap_size_total_bytes) + sum(discv5_worker_nodejs_heap_size_total_bytes) + sum(lodestar_historical_state_worker_nodejs_heap_size_total_bytes))", "hide": false, "interval": "", "legendFormat": "node allocated heap", @@ -191,7 +191,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "(sum(nodejs_heap_size_used_bytes) + sum(discv5_worker_nodejs_heap_size_used_bytes) + sum(network_worker_nodejs_heap_size_used_bytes)) \nor\n(sum(nodejs_heap_size_used_bytes) + sum(discv5_worker_nodejs_heap_size_used_bytes)) ", + "expr": "(sum(nodejs_heap_size_used_bytes) + sum(discv5_worker_nodejs_heap_size_used_bytes) + sum(network_worker_nodejs_heap_size_used_bytes) + sum(lodestar_historical_state_worker_nodejs_heap_size_used_bytes)) \nor\n(sum(nodejs_heap_size_used_bytes) + sum(discv5_worker_nodejs_heap_size_used_bytes) + sum(lodestar_historical_state_worker_nodejs_heap_size_used_bytes)) ", "hide": false, "interval": "", "legendFormat": "node used heap", @@ -205,7 +205,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "(sum(nodejs_external_memory_bytes) + sum(discv5_worker_nodejs_external_memory_bytes) + sum(network_worker_nodejs_external_memory_bytes))\nor\n(sum(nodejs_external_memory_bytes) + sum(discv5_worker_nodejs_external_memory_bytes))", + "expr": "(sum(nodejs_external_memory_bytes) + sum(discv5_worker_nodejs_external_memory_bytes) + sum(network_worker_nodejs_external_memory_bytes) + sum(lodestar_historical_state_worker_nodejs_external_memory_bytes))\nor\n(sum(nodejs_external_memory_bytes) + sum(discv5_worker_nodejs_external_memory_bytes) + sum(lodestar_historical_state_worker_nodejs_external_memory_bytes))", "hide": false, "interval": "", "legendFormat": "node external memory", @@ -291,7 +291,7 @@ "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", - "expr": "sum by (space) ({__name__=~\"nodejs_heap_space_size_used_bytes|network_worker_nodejs_heap_space_size_used_bytes|discv5_worker_nodejs_heap_space_size_used_bytes\")", + "expr": "sum by (space) ({__name__=~\"nodejs_heap_space_size_used_bytes|network_worker_nodejs_heap_space_size_used_bytes|discv5_worker_nodejs_heap_space_size_used_bytes|lodestar_historical_state_worker_nodejs_heap_space_size_used_bytes\")", "hide": false, "interval": "", "legendFormat": "{{space}}", @@ -305,7 +305,7 @@ }, "editorMode": "code", "exemplar": false, - "expr": "(nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"} + discv5_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"} + network_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"})\nor\n(nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"} + discv5_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"})", + "expr": "(nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"} + discv5_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"} + network_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"} + lodestar_historical_state_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"})\nor\n(nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"} + discv5_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"} + lodestar_historical_state_worker_nodejs_external_memory_bytes{job=~\"$beacon_job|beacon\"})", "hide": false, "interval": "", "legendFormat": "external_memory", @@ -786,6 +786,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -883,6 +884,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1000,6 +1002,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1027,6 +1030,222 @@ "x": 0, "y": 34 }, + "id": 562, + "options": { + "graph": {}, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "lodestar_historical_state_worker_nodejs_heap_space_size_used_bytes", + "interval": "", + "legendFormat": "{{space}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "lodestar_historical_state_worker_nodejs_external_memory_bytes", + "hide": false, + "interval": "", + "legendFormat": "external_memory", + "range": true, + "refId": "B" + } + ], + "title": "Historical State Worker Thread - Heap Allocations", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "GC Time", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "percentunit" + }, + "overrides": [ + { + "matcher": { + "id": "byFrameRefID", + "options": "C" + }, + "properties": [ + { + "id": "custom.axisPlacement", + "value": "right" + }, + { + "id": "custom.axisLabel", + "value": "GC Bytes" + }, + { + "id": "unit", + "value": "decbytes" + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 563, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "8.4.0-beta1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_worker_nodejs_gc_pause_seconds_total[$rate_interval])", + "hide": false, + "legendFormat": "{{gctype}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "rate(lodestar_historical_state_worker_nodejs_gc_reclaimed_bytes_total[$rate_interval])", + "format": "time_series", + "hide": false, + "legendFormat": "{{gctype}}", + "range": true, + "refId": "C" + } + ], + "title": "HIstorical State Worker Thread - GC pause time rate + reclaimed bytes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "opacity", + "hideFrom": { + "graph": false, + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 42 + }, "id": 542, "options": { "graph": {}, @@ -1097,6 +1316,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1143,7 +1363,7 @@ "h": 8, "w": 12, "x": 12, - "y": 34 + "y": 42 }, "id": 548, "options": { @@ -1199,7 +1419,7 @@ "h": 1, "w": 24, "x": 0, - "y": 42 + "y": 50 }, "id": 12, "panels": [], @@ -1239,6 +1459,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1368,7 +1589,7 @@ "h": 8, "w": 12, "x": 0, - "y": 43 + "y": 51 }, "id": 555, "options": { @@ -1483,6 +1704,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -1612,7 +1834,7 @@ "h": 8, "w": 12, "x": 12, - "y": 43 + "y": 51 }, "id": 559, "options": { @@ -1856,7 +2078,7 @@ "h": 8, "w": 12, "x": 0, - "y": 51 + "y": 59 }, "id": 560, "options": { @@ -2100,7 +2322,7 @@ "h": 8, "w": 12, "x": 12, - "y": 51 + "y": 59 }, "id": 561, "options": { @@ -2241,7 +2463,7 @@ "h": 8, "w": 12, "x": 0, - "y": 59 + "y": 67 }, "id": 6, "options": { @@ -2364,7 +2586,7 @@ "h": 8, "w": 12, "x": 12, - "y": 59 + "y": 67 }, "id": 42, "options": { @@ -2443,7 +2665,7 @@ "h": 8, "w": 12, "x": 12, - "y": 67 + "y": 75 }, "id": 268, "options": { @@ -2484,7 +2706,7 @@ "h": 1, "w": 24, "x": 0, - "y": 75 + "y": 83 }, "id": 104, "panels": [], @@ -2550,7 +2772,7 @@ "h": 6, "w": 12, "x": 0, - "y": 76 + "y": 84 }, "id": 102, "options": { @@ -2659,7 +2881,7 @@ "h": 6, "w": 12, "x": 12, - "y": 76 + "y": 84 }, "id": 172, "options": { @@ -2744,7 +2966,7 @@ "h": 6, "w": 12, "x": 0, - "y": 82 + "y": 90 }, "id": 171, "options": { @@ -2787,7 +3009,7 @@ "h": 6, "w": 12, "x": 12, - "y": 82 + "y": 90 }, "id": 99, "options": { @@ -2828,7 +3050,7 @@ "h": 1, "w": 24, "x": 0, - "y": 88 + "y": 96 }, "id": 367, "panels": [], @@ -2894,7 +3116,7 @@ "h": 8, "w": 12, "x": 0, - "y": 89 + "y": 97 }, "id": 353, "options": { @@ -2977,7 +3199,7 @@ "h": 8, "w": 12, "x": 12, - "y": 89 + "y": 97 }, "id": 355, "options": { @@ -3059,7 +3281,7 @@ "h": 8, "w": 12, "x": 0, - "y": 97 + "y": 105 }, "id": 357, "options": { @@ -3522,7 +3744,7 @@ "h": 8, "w": 12, "x": 12, - "y": 97 + "y": 105 }, "id": 359, "links": [], @@ -3665,7 +3887,7 @@ "h": 8, "w": 12, "x": 0, - "y": 105 + "y": 113 }, "id": 361, "options": { @@ -3774,7 +3996,7 @@ "h": 8, "w": 12, "x": 12, - "y": 105 + "y": 113 }, "id": 363, "options": { @@ -3866,7 +4088,7 @@ "h": 8, "w": 12, "x": 0, - "y": 113 + "y": 121 }, "id": 365, "options": { @@ -3919,7 +4141,7 @@ "h": 1, "w": 24, "x": 0, - "y": 121 + "y": 129 }, "id": 46, "panels": [], @@ -3985,7 +4207,7 @@ "h": 8, "w": 12, "x": 0, - "y": 122 + "y": 130 }, "id": 48, "options": { @@ -4112,7 +4334,7 @@ "h": 8, "w": 12, "x": 12, - "y": 122 + "y": 130 }, "id": 50, "options": { @@ -4155,7 +4377,7 @@ "h": 1, "w": 24, "x": 0, - "y": 130 + "y": 138 }, "id": 527, "panels": [], @@ -4221,7 +4443,7 @@ "h": 8, "w": 12, "x": 0, - "y": 131 + "y": 139 }, "id": 529, "options": { @@ -4366,7 +4588,7 @@ "h": 8, "w": 12, "x": 12, - "y": 131 + "y": 139 }, "id": 531, "options": { @@ -4410,7 +4632,7 @@ "h": 1, "w": 24, "x": 0, - "y": 139 + "y": 147 }, "id": 86, "panels": [], @@ -4475,7 +4697,7 @@ "h": 11, "w": 12, "x": 0, - "y": 140 + "y": 148 }, "id": 84, "options": { @@ -4558,7 +4780,7 @@ "h": 11, "w": 12, "x": 12, - "y": 140 + "y": 148 }, "id": 87, "options": { @@ -4639,7 +4861,7 @@ "h": 8, "w": 12, "x": 0, - "y": 151 + "y": 159 }, "id": 516, "options": { @@ -4730,7 +4952,7 @@ "h": 8, "w": 12, "x": 12, - "y": 151 + "y": 159 }, "id": 515, "options": { @@ -4809,7 +5031,7 @@ "h": 8, "w": 12, "x": 0, - "y": 159 + "y": 167 }, "id": 513, "options": { @@ -4888,7 +5110,7 @@ "h": 8, "w": 12, "x": 12, - "y": 159 + "y": 167 }, "id": 517, "options": { @@ -4928,7 +5150,7 @@ "h": 1, "w": 24, "x": 0, - "y": 167 + "y": 175 }, "id": 164, "panels": [], @@ -4995,7 +5217,7 @@ "h": 9, "w": 12, "x": 0, - "y": 168 + "y": 176 }, "id": 160, "options": { @@ -5079,7 +5301,7 @@ "h": 9, "w": 12, "x": 12, - "y": 168 + "y": 176 }, "id": 162, "options": { @@ -5163,7 +5385,7 @@ "h": 9, "w": 12, "x": 0, - "y": 177 + "y": 185 }, "id": 481, "options": { @@ -5247,7 +5469,7 @@ "h": 9, "w": 12, "x": 12, - "y": 177 + "y": 185 }, "id": 480, "options": { @@ -5354,7 +5576,7 @@ "h": 8, "w": 12, "x": 0, - "y": 186 + "y": 194 }, "id": 535, "options": { @@ -5459,7 +5681,7 @@ "h": 8, "w": 12, "x": 12, - "y": 186 + "y": 194 }, "id": 537, "options": { @@ -5540,7 +5762,7 @@ "h": 8, "w": 12, "x": 0, - "y": 194 + "y": 202 }, "id": 533, "options": {