diff --git a/bolt-sidecar/infra/grafana/provisioning/dashboards/dashboard.json b/bolt-sidecar/infra/grafana/provisioning/dashboards/dashboard.json index b7da46f8b..bfcecfb93 100644 --- a/bolt-sidecar/infra/grafana/provisioning/dashboards/dashboard.json +++ b/bolt-sidecar/infra/grafana/provisioning/dashboards/dashboard.json @@ -18,41 +18,254 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, + "id": 1, "links": [], "liveNow": false, "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "Sidecar Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bolt-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bolt-prometheus" + }, + "editorMode": "builder", + "expr": "bolt_sidecar_latest_head", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Latest slot", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bolt-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "green", + "value": 80 + } + ] + }, + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 5, + "y": 1 + }, + "id": 16, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": true + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bolt-prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(time() - container_start_time_seconds{name=~\"bolt-(sidecar|mev-boost|boost).*\"})/3600", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "{{name}}", + "range": false, + "refId": "A" + } + ], + "title": "Uptime", + "type": "gauge" + }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, + "description": "Tracks the reasons for rejecting an inclusion request.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 10, + "options": { + "displayLabels": ["name"], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "donut", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bolt-prometheus" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "bolt_sidecar_validation_errors", + "legendFormat": "{{type}}", + "range": true, + "refId": "A" + } + ], + "title": "Invalid transaction reasons", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bolt-prometheus" + }, + "description": "The number of preconfirmed transactions categorized by transaction type.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -83,7 +296,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 9 }, "id": 9, "options": { @@ -102,49 +315,53 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, "editorMode": "builder", "expr": "bolt_sidecar_transactions_preconfirmed", - "legendFormat": "__auto", + "legendFormat": "{{type}}", "range": true, "refId": "A" } ], - "title": "Transactions Preconfirmed", + "title": "Preconfirmed transactions", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, + "description": "The revenue from all of the preconfirmed transactions. This is \"gross\" revenue because it just adds all of the priority fees in those transactions, which may be partly split with the builder in case of a PBS block.", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "fixedColor": "semi-dark-blue", + "mode": "fixed" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "ETH", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -175,9 +392,9 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 9 }, - "id": 10, + "id": 17, "options": { "legend": { "calcs": [], @@ -194,49 +411,58 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, + "disableTextWrap": false, "editorMode": "builder", - "expr": "bolt_sidecar_validation_errors", - "legendFormat": "__auto", + "expr": "bolt_sidecar_gross_tip_revenue / 1000000000000000000", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{job}}", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false } ], - "title": "Invalid Transactions Reasons", + "title": "Gross tip revenue", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, + "description": "The number of inclusion requests that were accepted (i.e. committed to).", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "fixedColor": "green", + "mode": "fixed" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -263,9 +489,9 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 17 }, - "id": 2, + "id": 5, "options": { "legend": { "calcs": [], @@ -282,49 +508,53 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, "editorMode": "builder", - "expr": "bolt_sidecar_remote_blocks_proposed", - "legendFormat": "__auto", + "expr": "bolt_sidecar_inclusion_commitments_accepted", + "legendFormat": "requests", "range": true, "refId": "A" } ], - "title": "Remote Blocks Proposed", + "title": "Inclusion requests accepted", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, + "description": "The number of inclusion requests received.", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "fixedColor": "light-orange", + "mode": "fixed" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -355,7 +585,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 17 }, "id": 4, "options": { @@ -374,49 +604,56 @@ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, "editorMode": "builder", "expr": "bolt_sidecar_inclusion_commitments_received", - "legendFormat": "__auto", + "legendFormat": "requests", "range": true, "refId": "A" } ], - "title": "Inclusion Commitments Received", + "title": "Inclusion requests", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, + "description": "The number of PBS blocks proposed (as opposed to local fallback blocks).", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "fixedColor": "dark-blue", + "mode": "fixed" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "stepAfter", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -433,48 +670,19 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "{__name__=\"bolt_sidecar_local_blocks_proposed\", instance=\"172.16.0.25:9063\", job=\"mev-sidecar\"}" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 25 }, - "id": 8, + "id": 2, "options": { "legend": { "calcs": [], @@ -487,57 +695,57 @@ "sort": "none" } }, - "pluginVersion": "9.5.12", "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, "editorMode": "builder", - "exemplar": false, - "expr": "bolt_sidecar_local_blocks_proposed", - "instant": false, - "interval": "", - "legendFormat": "__auto", + "expr": "bolt_sidecar_remote_blocks_proposed", + "legendFormat": "blocks", "range": true, "refId": "A" } ], - "title": "Local Blocks Proposed", + "title": "PBS blocks proposed", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, + "description": "The number of fallback blocks proposed. Fallback blocks are only proposed in case of a PBS failure. If this number is high, something might be wrong.", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "fixedColor": "red", + "mode": "shades" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 9, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -554,6 +762,10 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] } @@ -564,9 +776,9 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 25 }, - "id": 5, + "id": 8, "options": { "legend": { "calcs": [], @@ -579,26 +791,145 @@ "sort": "none" } }, + "pluginVersion": "9.5.12", "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, + "disableTextWrap": false, "editorMode": "builder", - "expr": "bolt_sidecar_inclusion_commitments_accepted", - "legendFormat": "__auto", + "exemplar": false, + "expr": "bolt_sidecar_local_blocks_proposed", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "blocks", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Fallback blocks proposed", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 12, + "panels": [], + "title": "System Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bolt-prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 13, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bolt-prometheus" + }, + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{name=~\"bolt-sidecar.*|bolt-mev-boost.*|bolt-boost.*\"}[5m])) by (name) *100", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", "range": true, "refId": "A" } ], - "title": "Inclusion Commitments Accepted", + "title": "CPU usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, "fieldConfig": { "defaults": { @@ -606,26 +937,28 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -648,64 +981,91 @@ "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 12, + "w": 24, "x": 0, - "y": 24 + "y": 41 }, - "id": 6, + "id": 14, "options": { "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "right", "showLegend": true }, "tooltip": { - "mode": "single", + "mode": "multi", "sort": "none" } }, + "pluginVersion": "7.4.5", "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, - "editorMode": "builder", - "expr": "bolt_sidecar_http_requests_total", - "legendFormat": "__auto", + "editorMode": "code", + "expr": "sum(container_memory_rss{name=~\"bolt-sidecar.*|bolt-boost.*|bolt-mev-boost.*\"}) by (name)", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", "range": true, "refId": "A" } ], - "title": "Total HTTP Requests", + "title": "Memory usage", "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, + "description": "Total HTTP requests grouped by HTTP method & response code.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { - "fillOpacity": 80, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineWidth": 1 + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { @@ -727,45 +1087,64 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 24 + "x": 0, + "y": 49 }, - "id": 7, + "id": 6, "options": { - "bucketOffset": 0, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } }, "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, "editorMode": "builder", - "expr": "bolt_sidecar_http_requests_duration_seconds", - "legendFormat": "__auto", + "expr": "bolt_sidecar_http_requests_total", + "legendFormat": "{{method}} / {{status}}", "range": true, "refId": "A" } ], - "title": "HTTP Requests Durations in ms", - "type": "histogram" + "title": "Total HTTP requests", + "type": "timeseries" }, { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, + "description": "Request processing durations per HTTP method.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, + "custom": { + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", @@ -773,52 +1152,57 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 3, - "x": 0, - "y": 32 + "h": 8, + "w": 12, + "x": 12, + "y": 49 }, - "id": 3, + "id": 7, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "auto" + "bucketOffset": 0, + "combine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } }, - "pluginVersion": "9.5.12", "targets": [ { "datasource": { "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "uid": "bolt-prometheus" }, + "disableTextWrap": false, "editorMode": "builder", - "expr": "bolt_sidecar_latest_head", - "legendFormat": "__auto", + "expr": "sum by(method) (bolt_sidecar_http_requests_duration_seconds)", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "{{method}}", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false } ], - "title": "Latest Head Slot", - "type": "stat" + "title": "HTTP request durations", + "type": "histogram" } ], "refresh": "", - "schemaVersion": 38, - "style": "dark", + "schemaVersion": 39, "tags": [], "templating": { "list": [] @@ -829,8 +1213,8 @@ }, "timepicker": {}, "timezone": "", - "title": "Bolt Sidecar", - "uid": "e5960f6d-a1ed-4538-9c7c-3ecba4d4b4b1", - "version": 3, + "title": "bolt sidecar", + "uid": "bolt-prometheus", + "version": 24, "weekStart": "" } diff --git a/testnets/holesky/grafana/dashboards/dashboard.json b/testnets/holesky/grafana/dashboards/dashboard.json index 8823158a8..c5cae424b 100644 --- a/testnets/holesky/grafana/dashboards/dashboard.json +++ b/testnets/holesky/grafana/dashboards/dashboard.json @@ -18,41 +18,254 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, + "id": 1, "links": [], "liveNow": false, "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "Sidecar Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bolt-prometheus-holesky" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bolt-prometheus-holesky" + }, + "editorMode": "builder", + "expr": "bolt_sidecar_latest_head", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Latest slot", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bolt-prometheus-holesky" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "green", + "value": 80 + } + ] + }, + "unit": "h" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 7, + "x": 5, + "y": 1 + }, + "id": 16, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": true + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto", + "text": {} + }, + "pluginVersion": "11.1.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bolt-prometheus-holesky" + }, + "editorMode": "code", + "exemplar": false, + "expr": "(time() - container_start_time_seconds{name=~\"bolt-(sidecar|mev-boost|boost).*\"})/3600", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "{{name}}", + "range": false, + "refId": "A" + } + ], + "title": "Uptime", + "type": "gauge" + }, { "datasource": { "type": "prometheus", "uid": "bolt-prometheus-holesky" }, + "description": "Tracks the reasons for rejecting an inclusion request.", "fieldConfig": { "defaults": { "color": { "mode": "palette-classic" }, "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 10, + "options": { + "displayLabels": ["name"], + "legend": { + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "pieType": "donut", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bolt-prometheus-holesky" + }, + "editorMode": "builder", + "exemplar": false, + "expr": "bolt_sidecar_validation_errors", + "legendFormat": "{{type}}", + "range": true, + "refId": "A" + } + ], + "title": "Invalid transaction reasons", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bolt-prometheus-holesky" + }, + "description": "The number of preconfirmed transactions categorized by transaction type.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -83,7 +296,7 @@ "h": 8, "w": 12, "x": 0, - "y": 0 + "y": 9 }, "id": 9, "options": { @@ -106,12 +319,12 @@ }, "editorMode": "builder", "expr": "bolt_sidecar_transactions_preconfirmed", - "legendFormat": "__auto", + "legendFormat": "{{type}}", "range": true, "refId": "A" } ], - "title": "Transactions Preconfirmed", + "title": "Preconfirmed transactions", "type": "timeseries" }, { @@ -119,32 +332,36 @@ "type": "prometheus", "uid": "bolt-prometheus-holesky" }, + "description": "The revenue from all of the preconfirmed transactions. This is \"gross\" revenue because it just adds all of the priority fees in those transactions, which may be partly split with the builder in case of a PBS block.", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "fixedColor": "semi-dark-blue", + "mode": "fixed" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "ETH", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -175,9 +392,9 @@ "h": 8, "w": 12, "x": 12, - "y": 0 + "y": 9 }, - "id": 10, + "id": 17, "options": { "legend": { "calcs": [], @@ -196,14 +413,19 @@ "type": "prometheus", "uid": "bolt-prometheus-holesky" }, + "disableTextWrap": false, "editorMode": "builder", - "expr": "bolt_sidecar_validation_errors", - "legendFormat": "__auto", + "expr": "bolt_sidecar_gross_tip_revenue / 1000000000000000000", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{job}}", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false } ], - "title": "Invalid Transactions Reasons", + "title": "Gross tip revenue", "type": "timeseries" }, { @@ -211,32 +433,36 @@ "type": "prometheus", "uid": "bolt-prometheus-holesky" }, + "description": "The number of inclusion requests that were accepted (i.e. committed to).", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "fixedColor": "green", + "mode": "fixed" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -263,9 +489,9 @@ "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 17 }, - "id": 2, + "id": 5, "options": { "legend": { "calcs": [], @@ -285,13 +511,13 @@ "uid": "bolt-prometheus-holesky" }, "editorMode": "builder", - "expr": "bolt_sidecar_remote_blocks_proposed", - "legendFormat": "__auto", + "expr": "bolt_sidecar_inclusion_commitments_accepted", + "legendFormat": "requests", "range": true, "refId": "A" } ], - "title": "Remote Blocks Proposed", + "title": "Inclusion requests accepted", "type": "timeseries" }, { @@ -299,32 +525,36 @@ "type": "prometheus", "uid": "bolt-prometheus-holesky" }, + "description": "The number of inclusion requests received.", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "fixedColor": "light-orange", + "mode": "fixed" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -355,7 +585,7 @@ "h": 8, "w": 12, "x": 12, - "y": 8 + "y": 17 }, "id": 4, "options": { @@ -378,12 +608,12 @@ }, "editorMode": "builder", "expr": "bolt_sidecar_inclusion_commitments_received", - "legendFormat": "__auto", + "legendFormat": "requests", "range": true, "refId": "A" } ], - "title": "Inclusion Commitments Received", + "title": "Inclusion requests", "type": "timeseries" }, { @@ -391,32 +621,39 @@ "type": "prometheus", "uid": "bolt-prometheus-holesky" }, + "description": "The number of PBS blocks proposed (as opposed to local fallback blocks).", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "fixedColor": "dark-blue", + "mode": "fixed" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "stepAfter", + "lineStyle": { + "fill": "solid" + }, "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -433,48 +670,19 @@ { "color": "green", "value": null - }, - { - "color": "red", - "value": 80 } ] } }, - "overrides": [ - { - "__systemRef": "hideSeriesFrom", - "matcher": { - "id": "byNames", - "options": { - "mode": "exclude", - "names": [ - "{__name__=\"bolt_sidecar_local_blocks_proposed\", instance=\"172.16.0.25:9063\", job=\"bolt-sidecar\"}" - ], - "prefix": "All except:", - "readOnly": true - } - }, - "properties": [ - { - "id": "custom.hideFrom", - "value": { - "legend": false, - "tooltip": false, - "viz": true - } - } - ] - } - ] + "overrides": [] }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 16 + "y": 25 }, - "id": 8, + "id": 2, "options": { "legend": { "calcs": [], @@ -487,7 +695,6 @@ "sort": "none" } }, - "pluginVersion": "9.5.12", "targets": [ { "datasource": { @@ -495,16 +702,13 @@ "uid": "bolt-prometheus-holesky" }, "editorMode": "builder", - "exemplar": false, - "expr": "bolt_sidecar_local_blocks_proposed", - "instant": false, - "interval": "", - "legendFormat": "__auto", + "expr": "bolt_sidecar_remote_blocks_proposed", + "legendFormat": "blocks", "range": true, "refId": "A" } ], - "title": "Local Blocks Proposed", + "title": "PBS blocks proposed", "type": "timeseries" }, { @@ -512,32 +716,36 @@ "type": "prometheus", "uid": "bolt-prometheus-holesky" }, + "description": "The number of fallback blocks proposed. Fallback blocks are only proposed in case of a PBS failure. If this number is high, something might be wrong.", "fieldConfig": { "defaults": { "color": { - "mode": "palette-classic" + "fixedColor": "red", + "mode": "shades" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 9, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "stepAfter", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -554,6 +762,10 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] } @@ -564,9 +776,9 @@ "h": 8, "w": 12, "x": 12, - "y": 16 + "y": 25 }, - "id": 5, + "id": 8, "options": { "legend": { "calcs": [], @@ -579,20 +791,139 @@ "sort": "none" } }, + "pluginVersion": "9.5.12", "targets": [ { "datasource": { "type": "prometheus", "uid": "bolt-prometheus-holesky" }, + "disableTextWrap": false, "editorMode": "builder", - "expr": "bolt_sidecar_inclusion_commitments_accepted", - "legendFormat": "__auto", + "exemplar": false, + "expr": "bolt_sidecar_local_blocks_proposed", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "blocks", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Fallback blocks proposed", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 12, + "panels": [], + "title": "System Metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "bolt-prometheus-holesky" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 13, + "options": { + "legend": { + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "7.4.5", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "bolt-prometheus-holesky" + }, + "editorMode": "code", + "expr": "sum(rate(container_cpu_usage_seconds_total{name=~\"bolt-sidecar.*|bolt-mev-boost.*|bolt-boost.*\"}[5m])) by (name) *100", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", "range": true, "refId": "A" } ], - "title": "Inclusion Commitments Accepted", + "title": "CPU usage", "type": "timeseries" }, { @@ -606,26 +937,28 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, "axisCenteredZero": false, "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", - "fillOpacity": 0, + "fillOpacity": 10, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, "scaleDistribution": { "type": "linear" }, - "showPoints": "auto", + "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", @@ -648,43 +981,47 @@ "value": 80 } ] - } + }, + "unit": "bytes" }, "overrides": [] }, "gridPos": { "h": 8, - "w": 12, + "w": 24, "x": 0, - "y": 24 + "y": 41 }, - "id": 6, + "id": 14, "options": { "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", + "calcs": ["mean", "max"], + "displayMode": "table", + "placement": "right", "showLegend": true }, "tooltip": { - "mode": "single", + "mode": "multi", "sort": "none" } }, + "pluginVersion": "7.4.5", "targets": [ { "datasource": { "type": "prometheus", "uid": "bolt-prometheus-holesky" }, - "editorMode": "builder", - "expr": "bolt_sidecar_http_requests_total", - "legendFormat": "__auto", + "editorMode": "code", + "expr": "sum(container_memory_rss{name=~\"bolt-sidecar.*|bolt-boost.*|bolt-mev-boost.*\"}) by (name)", + "hide": false, + "interval": "", + "legendFormat": "{{name}}", "range": true, "refId": "A" } ], - "title": "Total HTTP Requests", + "title": "Memory usage", "type": "timeseries" }, { @@ -692,20 +1029,43 @@ "type": "prometheus", "uid": "bolt-prometheus-holesky" }, + "description": "Total HTTP requests grouped by HTTP method & response code.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, "custom": { - "fillOpacity": 80, + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, - "lineWidth": 1 + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } }, "mappings": [], "thresholds": { @@ -727,17 +1087,20 @@ "gridPos": { "h": 8, "w": 12, - "x": 12, - "y": 24 + "x": 0, + "y": 49 }, - "id": 7, + "id": 6, "options": { - "bucketOffset": 0, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } }, "targets": [ @@ -747,25 +1110,41 @@ "uid": "bolt-prometheus-holesky" }, "editorMode": "builder", - "expr": "bolt_sidecar_http_requests_duration_seconds", - "legendFormat": "__auto", + "expr": "bolt_sidecar_http_requests_total", + "legendFormat": "{{method}} / {{status}}", "range": true, "refId": "A" } ], - "title": "HTTP Requests Durations in ms", - "type": "histogram" + "title": "Total HTTP requests", + "type": "timeseries" }, { "datasource": { "type": "prometheus", "uid": "bolt-prometheus-holesky" }, + "description": "Request processing durations per HTTP method.", "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, + "custom": { + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 0, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "decimals": 0, "mappings": [], "thresholds": { "mode": "absolute", @@ -773,52 +1152,57 @@ { "color": "green", "value": null + }, + { + "color": "red", + "value": 80 } ] - } + }, + "unit": "s" }, "overrides": [] }, "gridPos": { - "h": 5, - "w": 3, - "x": 0, - "y": 32 + "h": 8, + "w": 12, + "x": 12, + "y": 49 }, - "id": 3, + "id": 7, "options": { - "colorMode": "value", - "graphMode": "none", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "auto" + "bucketOffset": 0, + "combine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } }, - "pluginVersion": "9.5.12", "targets": [ { "datasource": { "type": "prometheus", "uid": "bolt-prometheus-holesky" }, + "disableTextWrap": false, "editorMode": "builder", - "expr": "bolt_sidecar_latest_head", - "legendFormat": "__auto", + "expr": "sum by(method) (bolt_sidecar_http_requests_duration_seconds)", + "fullMetaSearch": false, + "includeNullMetadata": false, + "legendFormat": "{{method}}", "range": true, - "refId": "A" + "refId": "A", + "useBackend": false } ], - "title": "Latest Head Slot", - "type": "stat" + "title": "HTTP request durations", + "type": "histogram" } ], "refresh": "", - "schemaVersion": 38, - "style": "dark", + "schemaVersion": 39, "tags": [], "templating": { "list": [] @@ -829,8 +1213,8 @@ }, "timepicker": {}, "timezone": "", - "title": "bolt-prometheus-holesky", + "title": "bolt sidecar Holesky", "uid": "bolt-prometheus-holesky", - "version": 3, + "version": 24, "weekStart": "" }