From ea186a20d030048df232b0017dc1bf297d0dfd71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Arjovsky?= Date: Mon, 22 Jul 2024 18:06:07 +0200 Subject: [PATCH] feat: fork choice metrics (#1232) --- .../fork_choice/fork_choice.ex | 17 +- lib/lambda_ethereum_consensus/metrics.ex | 6 + .../state_transition/operations.ex | 33 +- .../state_transition/state_transition.ex | 51 +- lib/lambda_ethereum_consensus/telemetry.ex | 5 +- .../grafana/provisioning/dashboards/home.json | 753 ++++++++++-------- 6 files changed, 499 insertions(+), 366 deletions(-) diff --git a/lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex b/lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex index c0869f1f3..b07d5f064 100644 --- a/lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex +++ b/lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex @@ -197,12 +197,13 @@ defmodule LambdaEthereumConsensus.ForkChoice do end end - @spec apply_handler(any(), any(), any()) :: any() - defp apply_handler(iter, state, handler) do - iter - |> Enum.reduce_while({:ok, state}, fn - x, {:ok, st} -> {:cont, handler.(st, x)} - _, {:error, _} = err -> {:halt, err} + defp apply_handler(iter, name, state, handler) do + Metrics.span_operation(name, nil, nil, fn -> + iter + |> Enum.reduce_while({:ok, state}, fn + x, {:ok, st} -> {:cont, handler.(st, x)} + _, {:error, _} = err -> {:halt, err} + end) end) end @@ -212,11 +213,11 @@ defmodule LambdaEthereumConsensus.ForkChoice do # process block attestations {:ok, new_store} <- signed_block.message.body.attestations - |> apply_handler(new_store, &Handlers.on_attestation(&1, &2, true)), + |> apply_handler(:attestations, new_store, &Handlers.on_attestation(&1, &2, true)), # process block attester slashings {:ok, new_store} <- signed_block.message.body.attester_slashings - |> apply_handler(new_store, &Handlers.on_attester_slashing/2) do + |> apply_handler(:attester_slashings, new_store, &Handlers.on_attester_slashing/2) do Handlers.prune_checkpoint_states(new_store) {:ok, new_store} end diff --git a/lib/lambda_ethereum_consensus/metrics.ex b/lib/lambda_ethereum_consensus/metrics.ex index 764b77cdd..b46388925 100644 --- a/lib/lambda_ethereum_consensus/metrics.ex +++ b/lib/lambda_ethereum_consensus/metrics.ex @@ -104,6 +104,12 @@ defmodule LambdaEthereumConsensus.Metrics do end end + def span_operation(handler, transition, operation, f) do + :telemetry.span([:fork_choice, :latency], %{}, fn -> + {f.(), %{handler: handler, transition: transition, operation: operation}} + end) + end + def handler_span(module, action, f) do :telemetry.span([:libp2pport, :handler], %{}, fn -> {f.(), %{module: module, action: action}} diff --git a/lib/lambda_ethereum_consensus/state_transition/operations.ex b/lib/lambda_ethereum_consensus/state_transition/operations.ex index 7a2e46bcd..76acd20e9 100644 --- a/lib/lambda_ethereum_consensus/state_transition/operations.ex +++ b/lib/lambda_ethereum_consensus/state_transition/operations.ex @@ -3,6 +3,7 @@ defmodule LambdaEthereumConsensus.StateTransition.Operations do This module contains functions for handling state transition """ + alias LambdaEthereumConsensus.Metrics alias LambdaEthereumConsensus.StateTransition.Accessors alias LambdaEthereumConsensus.StateTransition.Math alias LambdaEthereumConsensus.StateTransition.Misc @@ -919,19 +920,31 @@ defmodule LambdaEthereumConsensus.StateTransition.Operations do # Ensure that outstanding deposits are processed up to the maximum number of deposits with :ok <- verify_deposits(state, body) do {:ok, state} - |> for_ops(body.proposer_slashings, &process_proposer_slashing/2) - |> for_ops(body.attester_slashings, &process_attester_slashing/2) - |> Utils.map_ok(&process_attestation_batch(&1, body.attestations)) - |> for_ops(body.deposits, &process_deposit/2) - |> for_ops(body.voluntary_exits, &process_voluntary_exit/2) - |> for_ops(body.bls_to_execution_changes, &process_bls_to_execution_change/2) + |> for_ops(:proposer_slashing, body.proposer_slashings, &process_proposer_slashing/2) + |> for_ops(:attester_slashing, body.attester_slashings, &process_attester_slashing/2) + |> apply_op(:attestation_batch, &process_attestation_batch(&1, body.attestations)) + |> for_ops(:deposit, body.deposits, &process_deposit/2) + |> for_ops(:voluntary_exit, body.voluntary_exits, &process_voluntary_exit/2) + |> for_ops( + :bls_to_execution_change, + body.bls_to_execution_changes, + &process_bls_to_execution_change/2 + ) end end - defp for_ops(acc, operations, func) do - Enum.reduce_while(operations, acc, fn - operation, {:ok, state} -> {:cont, func.(state, operation)} - _, {:error, reason} -> {:halt, {:error, reason}} + defp apply_op(acc, op_name, func) do + Metrics.span_operation(:on_block, :process_block_operations, op_name, fn -> + Utils.map_ok(acc, func) + end) + end + + defp for_ops(acc, op_name, operations, func) do + Metrics.span_operation(:on_block, :process_block_operations, op_name, fn -> + Enum.reduce_while(operations, acc, fn + operation, {:ok, state} -> {:cont, func.(state, operation)} + _, {:error, reason} -> {:halt, {:error, reason}} + end) end) end diff --git a/lib/lambda_ethereum_consensus/state_transition/state_transition.ex b/lib/lambda_ethereum_consensus/state_transition/state_transition.ex index 180769415..486bbb190 100644 --- a/lib/lambda_ethereum_consensus/state_transition/state_transition.ex +++ b/lib/lambda_ethereum_consensus/state_transition/state_transition.ex @@ -4,6 +4,7 @@ defmodule LambdaEthereumConsensus.StateTransition do """ require Logger + alias LambdaEthereumConsensus.Metrics alias LambdaEthereumConsensus.StateTransition.Accessors alias LambdaEthereumConsensus.StateTransition.EpochProcessing alias LambdaEthereumConsensus.StateTransition.Misc @@ -108,17 +109,23 @@ defmodule LambdaEthereumConsensus.StateTransition do state |> EpochProcessing.process_justification_and_finalization() - |> map_ok(&EpochProcessing.process_inactivity_updates/1) - |> map_ok(&EpochProcessing.process_rewards_and_penalties/1) - |> map_ok(&EpochProcessing.process_registry_updates/1) - |> map_ok(&EpochProcessing.process_slashings/1) - |> map_ok(&EpochProcessing.process_eth1_data_reset/1) - |> map_ok(&EpochProcessing.process_effective_balance_updates/1) - |> map_ok(&EpochProcessing.process_slashings_reset/1) - |> map_ok(&EpochProcessing.process_randao_mixes_reset/1) - |> map_ok(&EpochProcessing.process_historical_summaries_update/1) - |> map_ok(&EpochProcessing.process_participation_flag_updates/1) - |> map_ok(&EpochProcessing.process_sync_committee_updates/1) + |> epoch_op(:inactivity_updates, &EpochProcessing.process_inactivity_updates/1) + |> epoch_op(:rewards_and_penalties, &EpochProcessing.process_rewards_and_penalties/1) + |> epoch_op(:registry_updates, &EpochProcessing.process_registry_updates/1) + |> epoch_op(:slashings, &EpochProcessing.process_slashings/1) + |> epoch_op(:eth1_data_reset, &EpochProcessing.process_eth1_data_reset/1) + |> epoch_op(:effective_balance_updates, &EpochProcessing.process_effective_balance_updates/1) + |> epoch_op(:slashings_reset, &EpochProcessing.process_slashings_reset/1) + |> epoch_op(:randao_mixes_reset, &EpochProcessing.process_randao_mixes_reset/1) + |> epoch_op( + :historical_summaries_update, + &EpochProcessing.process_historical_summaries_update/1 + ) + |> epoch_op( + :participation_flag_updates, + &EpochProcessing.process_participation_flag_updates/1 + ) + |> epoch_op(:sync_committee_updates, &EpochProcessing.process_sync_committee_updates/1) |> tap(fn _ -> end_time = System.monotonic_time(:millisecond) Logger.debug("[Epoch processing] took #{end_time - start_time} ms") @@ -136,16 +143,26 @@ defmodule LambdaEthereumConsensus.StateTransition do start_time = System.monotonic_time(:millisecond) {:ok, state} - |> map_ok(&Operations.process_block_header(&1, block)) - |> map_ok(&Operations.process_withdrawals(&1, block.body.execution_payload)) - |> map_ok(&Operations.process_execution_payload(&1, block.body)) - |> map_ok(&Operations.process_randao(&1, block.body)) - |> map_ok(&Operations.process_eth1_data(&1, block.body)) + |> block_op(:block_header, &Operations.process_block_header(&1, block)) + |> block_op(:withdrawals, &Operations.process_withdrawals(&1, block.body.execution_payload)) + |> block_op(:execution_payload, &Operations.process_execution_payload(&1, block.body)) + |> block_op(:randao, &Operations.process_randao(&1, block.body)) + |> block_op(:eth1_data, &Operations.process_eth1_data(&1, block.body)) |> map_ok(&Operations.process_operations(&1, block.body)) - |> map_ok(&Operations.process_sync_aggregate(&1, block.body.sync_aggregate)) + |> block_op( + :sync_aggregate, + &Operations.process_sync_aggregate(&1, block.body.sync_aggregate) + ) |> tap(fn _ -> end_time = System.monotonic_time(:millisecond) Logger.debug("[Block processing] took #{end_time - start_time} ms") end) end + + def block_op(state, operation, f), do: apply_op(state, :process_block, operation, f) + def epoch_op(state, operation, f), do: apply_op(state, :epoch, operation, f) + + def apply_op(state, transition, operation, f) do + Metrics.span_operation(:on_block, transition, operation, fn -> map_ok(state, f) end) + end end diff --git a/lib/lambda_ethereum_consensus/telemetry.ex b/lib/lambda_ethereum_consensus/telemetry.ex index 5ba4709d0..9971296e7 100644 --- a/lib/lambda_ethereum_consensus/telemetry.ex +++ b/lib/lambda_ethereum_consensus/telemetry.ex @@ -144,8 +144,11 @@ defmodule LambdaEthereumConsensus.Telemetry do unit: {:native, :millisecond}, tags: [:module, :action] ), - # ForkChoice Metrics + last_value("fork_choice.latency.stop.duration", + unit: {:native, :millisecond}, + tags: [:handler, :transition, :operation] + ), last_value("fork_choice.recompute_head.stop.duration", unit: {:native, :millisecond} ), diff --git a/metrics/grafana/provisioning/dashboards/home.json b/metrics/grafana/provisioning/dashboards/home.json index 78abf0b95..13ec76b2e 100644 --- a/metrics/grafana/provisioning/dashboards/home.json +++ b/metrics/grafana/provisioning/dashboards/home.json @@ -62,7 +62,7 @@ "x": 0, "y": 0 }, - "id": 38, + "id": 39, "options": { "reduceOptions": { "values": false, @@ -116,7 +116,7 @@ "uid": "PBFA97CFB590B2093", "type": "prometheus" }, - "id": 37, + "id": 38, "targets": [ { "refId": "A", @@ -190,7 +190,7 @@ "w": 12, "h": 8 }, - "id": 36, + "id": 37, "targets": [ { "datasource": { @@ -265,7 +265,7 @@ "uid": "PBFA97CFB590B2093", "type": "prometheus" }, - "id": 35, + "id": 36, "targets": [ { "refId": "A", @@ -339,7 +339,7 @@ "w": 12, "h": 8 }, - "id": 34, + "id": 35, "targets": [ { "datasource": { @@ -414,7 +414,7 @@ "uid": "PBFA97CFB590B2093", "type": "prometheus" }, - "id": 33, + "id": 34, "targets": [ { "refId": "A", @@ -488,7 +488,7 @@ "w": 12, "h": 8 }, - "id": 32, + "id": 33, "targets": [ { "datasource": { @@ -550,89 +550,6 @@ }, "pluginVersion": "11.0.0" }, - { - "datasource": { - "uid": "PBFA97CFB590B2093", - "type": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 31, - "options": { - "nodes": {}, - "edges": {} - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "blocks_status_total", - "format": "table", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A", - "useBackend": false, - "exemplar": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "blocks_relationship_total", - "format": "table", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "B", - "useBackend": false, - "exemplar": false - } - ], - "title": "Blockchain View", - "transformations": [ - { - "id": "filterByValue", - "options": { - "filters": [ - { - "fieldName": "Value #A", - "config": { - "id": "equal", - "options": { - "value": 0 - } - } - } - ], - "type": "exclude", - "match": "any" - }, - "filter": { - "id": "byRefId", - "options": "A" - }, - "topic": "series" - } - ], - "type": "nodeGraph" - }, { "datasource": { "type": "prometheus", @@ -641,49 +558,51 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } }, + "mappings": [], "unit": "ms" }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 12, "w": 12, "x": 0, "y": 0 }, - "id": 30, + "id": 32, "options": { - "displayMode": "gradient", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", "reduceOptions": { "calcs": [ - "mean" + "lastNotNull" ], "fields": "", "values": false }, - "showUnfilled": true, - "valueMode": "color" + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "9.5.1", "targets": [ { "datasource": { @@ -691,14 +610,38 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "builder", - "expr": "db_latency_stop_duration", - "legendFormat": "{{module}} - {{action}}", + "expr": "fork_choice_latency_stop_duration{handler=\"on_block\"}", + "legendFormat": "{{handler}} - {{transition}} - {{operation}}", "range": true, - "refId": "Mean time per db operation" + "refId": "on_block" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "fork_choice_recompute_head_stop_duration", + "hide": false, + "legendFormat": "recompute_head", + "range": true, + "refId": "recompute head" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "fork_choice_latency_stop_duration{handler!=\"on_block\"}", + "hide": false, + "legendFormat": "{{handler}}", + "range": true, + "refId": "Other handlers" } ], - "title": "Mean time per db operation", - "type": "bargauge" + "title": "Fork choice handler times", + "type": "piechart" }, { "datasource": { @@ -834,77 +777,16 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 6, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 12 }, - "id": 1, + "id": 31, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "edges": {}, + "nodes": {} }, "targets": [ { @@ -914,18 +796,64 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "db_size_total", + "exemplar": false, + "expr": "blocks_status_total", + "format": "table", "fullMetaSearch": false, "includeNullMetadata": true, "instant": true, "legendFormat": "__auto", - "range": true, + "range": false, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "blocks_relationship_total", + "format": "table", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B", + "useBackend": false } ], - "title": "Database size", - "type": "timeseries" + "title": "Blockchain View", + "transformations": [ + { + "filter": { + "id": "byRefId", + "options": "A" + }, + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "equal", + "options": { + "value": 0 + } + }, + "fieldName": "Value #A" + } + ], + "match": "any", + "type": "exclude" + }, + "topic": "series" + } + ], + "type": "nodeGraph" }, { "datasource": { @@ -1038,8 +966,166 @@ "useBackend": false } ], - "title": "P2P Requests", - "type": "timeseries" + "title": "P2P Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 14, + "maxDataPoints": 25, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "show": true, + "showLegend": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "show": true, + "sort": "none", + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "9.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "network_pubsub_topic_active_active", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{topic}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "PubSub topics joined", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 30, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "color" + }, + "pluginVersion": "9.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "db_latency_stop_duration", + "legendFormat": "{{module}} - {{action}}", + "range": true, + "refId": "Mean time per db operation" + } + ], + "title": "Mean time per db operation", + "type": "bargauge" }, { "datasource": { @@ -1066,8 +1152,8 @@ "viz": false }, "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 1, + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -1094,27 +1180,25 @@ "value": 80 } ] - }, - "unit": "ms" + } }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 6, "w": 12, - "x": 0, - "y": 16 + "x": 12, + "y": 24 }, - "id": 27, + "id": 11, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { - "maxHeight": 600, "mode": "single", "sort": "none" } @@ -1125,20 +1209,28 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "fork_choice_recompute_head_stop_duration", - "fullMetaSearch": false, + "editorMode": "code", + "expr": "rate(peers_challenge_count{result=\"failed\"}[$__rate_interval])", + "instant": false, + "legendFormat": "failed", + "range": true, + "refId": "Challenge failed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(peers_challenge_count{result=\"passed\"}[$__rate_interval])", "hide": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "__auto", + "instant": false, + "legendFormat": "passed", "range": true, - "refId": "A", - "useBackend": false + "refId": "Challenge passed" } ], - "title": "ForkChoice RecomputeHead Duration", + "title": "Peer challenges", "type": "timeseries" }, { @@ -1148,15 +1240,52 @@ }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } }, "overrides": [] @@ -1165,50 +1294,21 @@ "h": 6, "w": 12, "x": 12, - "y": 18 + "y": 30 }, - "id": 14, - "maxDataPoints": 25, + "id": 15, "options": { - "calculate": false, - "cellGap": 1, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Oranges", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "show": true, "showLegend": true }, - "rowsFrame": { - "layout": "auto" - }, "tooltip": { "mode": "single", - "show": true, - "sort": "none", - "yHistogram": false - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false + "sort": "none" } }, - "pluginVersion": "9.5.1", "targets": [ { "datasource": { @@ -1218,7 +1318,7 @@ "disableTextWrap": false, "editorMode": "code", "exemplar": false, - "expr": "network_pubsub_topic_active_active", + "expr": "network_pubsub_topics_graft_count{} - network_pubsub_topics_prune_count{}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -1229,8 +1329,8 @@ "useBackend": false } ], - "title": "PubSub topics joined", - "type": "heatmap" + "title": "Grafted", + "type": "timeseries" }, { "datasource": { @@ -1245,7 +1345,7 @@ "custom": { "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "Bytes", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -1257,8 +1357,8 @@ "viz": false }, "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 1, + "lineWidth": 1, + "pointSize": 6, "scaleDistribution": { "type": "linear" }, @@ -1286,7 +1386,7 @@ } ] }, - "unit": "ms" + "unit": "decbytes" }, "overrides": [] }, @@ -1294,9 +1394,9 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 35 }, - "id": 25, + "id": 1, "options": { "legend": { "calcs": [], @@ -1305,7 +1405,6 @@ "showLegend": false }, "tooltip": { - "maxHeight": 600, "mode": "single", "sort": "none" } @@ -1318,9 +1417,8 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "db_latency_stop_duration{module=\"fork_choice\", action=\"persist\"}", + "expr": "db_size_total", "fullMetaSearch": false, - "hide": false, "includeNullMetadata": true, "instant": true, "legendFormat": "__auto", @@ -1329,7 +1427,7 @@ "useBackend": false } ], - "title": "ForkChoice Persisting Store Duration", + "title": "Database size", "type": "timeseries" }, { @@ -1393,9 +1491,9 @@ "h": 6, "w": 12, "x": 12, - "y": 24 + "y": 36 }, - "id": 11, + "id": 16, "options": { "legend": { "calcs": [], @@ -1414,28 +1512,21 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "disableTextWrap": false, "editorMode": "code", - "expr": "rate(peers_challenge_count{result=\"failed\"}[$__rate_interval])", - "instant": false, - "legendFormat": "failed", - "range": true, - "refId": "Challenge failed" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "rate(peers_challenge_count{result=\"passed\"}[$__rate_interval])", - "hide": false, + "exemplar": false, + "expr": "rate(network_pubsub_topics_deliver_message_count{}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, "instant": false, - "legendFormat": "passed", + "interval": "", + "legendFormat": "{{topic}}", "range": true, - "refId": "Challenge passed" + "refId": "A", + "useBackend": false } ], - "title": "Peer challenges", + "title": "Deliver Messages", "type": "timeseries" }, { @@ -1499,9 +1590,9 @@ "h": 6, "w": 12, "x": 12, - "y": 30 + "y": 42 }, - "id": 15, + "id": 19, "options": { "legend": { "calcs": [], @@ -1523,7 +1614,7 @@ "disableTextWrap": false, "editorMode": "code", "exemplar": false, - "expr": "network_pubsub_topics_graft_count{} - network_pubsub_topics_prune_count{}", + "expr": "rate(network_pubsub_topics_un_deliverable_message_count{}[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -1534,7 +1625,7 @@ "useBackend": false } ], - "title": "Grafted", + "title": "Undeliverable Messages", "type": "timeseries" }, { @@ -1599,9 +1690,9 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 43 }, - "id": 28, + "id": 27, "options": { "legend": { "calcs": [], @@ -1623,7 +1714,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "db_latency_stop_duration{module=\"subnet\", action=\"persist\"}", + "expr": "fork_choice_recompute_head_stop_duration", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -1634,7 +1725,7 @@ "useBackend": false } ], - "title": "SubnetInfo Persisting Duration", + "title": "ForkChoice RecomputeHead Duration", "type": "timeseries" }, { @@ -1662,8 +1753,8 @@ "viz": false }, "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, + "lineWidth": 2, + "pointSize": 1, "scaleDistribution": { "type": "linear" }, @@ -1690,25 +1781,27 @@ "value": 80 } ] - } + }, + "unit": "ms" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, - "x": 12, - "y": 36 + "x": 0, + "y": 51 }, - "id": 16, + "id": 25, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" } @@ -1720,20 +1813,19 @@ "uid": "PBFA97CFB590B2093" }, "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "rate(network_pubsub_topics_deliver_message_count{}[$__rate_interval])", + "editorMode": "builder", + "expr": "db_latency_stop_duration{module=\"fork_choice\", action=\"persist\"}", "fullMetaSearch": false, + "hide": false, "includeNullMetadata": true, - "instant": false, - "interval": "", - "legendFormat": "{{topic}}", + "instant": true, + "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false } ], - "title": "Deliver Messages", + "title": "ForkChoice Persisting Store Duration", "type": "timeseries" }, { @@ -1798,9 +1890,9 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 59 }, - "id": 29, + "id": 28, "options": { "legend": { "calcs": [], @@ -1822,7 +1914,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "db_latency_stop_duration{module=\"subnet\", action=\"fetch\"}", + "expr": "db_latency_stop_duration{module=\"subnet\", action=\"persist\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -1833,7 +1925,7 @@ "useBackend": false } ], - "title": "SubnetInfo Fetching Duration", + "title": "SubnetInfo Persisting Duration", "type": "timeseries" }, { @@ -1861,8 +1953,8 @@ "viz": false }, "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, + "lineWidth": 2, + "pointSize": 1, "scaleDistribution": { "type": "linear" }, @@ -1889,25 +1981,27 @@ "value": 80 } ] - } + }, + "unit": "ms" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, - "x": 12, - "y": 42 + "x": 0, + "y": 67 }, - "id": 19, + "id": 29, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" } @@ -1919,20 +2013,19 @@ "uid": "PBFA97CFB590B2093" }, "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "rate(network_pubsub_topics_un_deliverable_message_count{}[$__rate_interval])", + "editorMode": "builder", + "expr": "db_latency_stop_duration{module=\"subnet\", action=\"fetch\"}", "fullMetaSearch": false, + "hide": false, "includeNullMetadata": true, - "instant": false, - "interval": "", - "legendFormat": "{{topic}}", + "instant": true, + "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false } ], - "title": "Undeliverable Messages", + "title": "SubnetInfo Fetching Duration", "type": "timeseries" }, { @@ -1997,7 +2090,7 @@ "h": 8, "w": 12, "x": 0, - "y": 48 + "y": 75 }, "id": 26, "options": { @@ -2077,7 +2170,7 @@ "h": 6, "w": 12, "x": 0, - "y": 56 + "y": 83 }, "hideTimeOverride": true, "id": 24, @@ -2173,7 +2266,7 @@ "h": 6, "w": 12, "x": 0, - "y": 62 + "y": 89 }, "id": 6, "options": { @@ -2282,7 +2375,7 @@ "h": 6, "w": 24, "x": 0, - "y": 68 + "y": 95 }, "id": 2, "options": { @@ -2447,7 +2540,7 @@ "h": 6, "w": 12, "x": 0, - "y": 74 + "y": 101 }, "id": 3, "options": { @@ -2558,7 +2651,7 @@ "h": 6, "w": 12, "x": 12, - "y": 74 + "y": 101 }, "id": 12, "options": { @@ -2666,7 +2759,7 @@ "h": 6, "w": 5, "x": 0, - "y": 80 + "y": 107 }, "id": 7, "options": { @@ -2726,7 +2819,7 @@ "h": 6, "w": 7, "x": 5, - "y": 80 + "y": 107 }, "id": 5, "options": { @@ -2826,7 +2919,7 @@ "h": 6, "w": 12, "x": 0, - "y": 86 + "y": 113 }, "id": 13, "options": { @@ -2925,7 +3018,7 @@ "h": 6, "w": 12, "x": 0, - "y": 92 + "y": 119 }, "id": 17, "options": { @@ -3024,7 +3117,7 @@ "h": 6, "w": 12, "x": 0, - "y": 98 + "y": 125 }, "id": 21, "options": { @@ -3148,7 +3241,7 @@ "h": 6, "w": 12, "x": 0, - "y": 104 + "y": 131 }, "id": 20, "options": { @@ -3247,7 +3340,7 @@ "h": 6, "w": 12, "x": 0, - "y": 110 + "y": 137 }, "id": 18, "options": { @@ -3301,6 +3394,6 @@ "timezone": "", "title": "Node", "uid": "90EXFQnIk", - "version": 3, + "version": 21, "weekStart": "" } \ No newline at end of file