From d960fbe749cfe1c31fde1a5081e86380d98d053d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Arjovsky?= Date: Fri, 19 Jul 2024 11:58:23 +0200 Subject: [PATCH 1/2] Add metrics for block processing, its subsections, attestation and attester slashing processing --- .../fork_choice/fork_choice.ex | 17 ++++--- lib/lambda_ethereum_consensus/metrics.ex | 6 +++ .../state_transition/operations.ex | 33 ++++++++---- .../state_transition/state_transition.ex | 51 ++++++++++++------- lib/lambda_ethereum_consensus/telemetry.ex | 4 ++ 5 files changed, 76 insertions(+), 35 deletions(-) diff --git a/lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex b/lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex index c0869f1f3..b07d5f064 100644 --- a/lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex +++ b/lib/lambda_ethereum_consensus/fork_choice/fork_choice.ex @@ -197,12 +197,13 @@ defmodule LambdaEthereumConsensus.ForkChoice do end end - @spec apply_handler(any(), any(), any()) :: any() - defp apply_handler(iter, state, handler) do - iter - |> Enum.reduce_while({:ok, state}, fn - x, {:ok, st} -> {:cont, handler.(st, x)} - _, {:error, _} = err -> {:halt, err} + defp apply_handler(iter, name, state, handler) do + Metrics.span_operation(name, nil, nil, fn -> + iter + |> Enum.reduce_while({:ok, state}, fn + x, {:ok, st} -> {:cont, handler.(st, x)} + _, {:error, _} = err -> {:halt, err} + end) end) end @@ -212,11 +213,11 @@ defmodule LambdaEthereumConsensus.ForkChoice do # process block attestations {:ok, new_store} <- signed_block.message.body.attestations - |> apply_handler(new_store, &Handlers.on_attestation(&1, &2, true)), + |> apply_handler(:attestations, new_store, &Handlers.on_attestation(&1, &2, true)), # process block attester slashings {:ok, new_store} <- signed_block.message.body.attester_slashings - |> apply_handler(new_store, &Handlers.on_attester_slashing/2) do + |> apply_handler(:attester_slashings, new_store, &Handlers.on_attester_slashing/2) do Handlers.prune_checkpoint_states(new_store) {:ok, new_store} end diff --git a/lib/lambda_ethereum_consensus/metrics.ex b/lib/lambda_ethereum_consensus/metrics.ex index 109e75f3a..f4d303463 100644 --- a/lib/lambda_ethereum_consensus/metrics.ex +++ b/lib/lambda_ethereum_consensus/metrics.ex @@ -120,6 +120,12 @@ defmodule LambdaEthereumConsensus.Metrics do end end + def span_operation(handler, transition, operation, f) do + :telemetry.span([:fork_choice, :latency], %{}, fn -> + {f.(), %{handler: handler, transition: transition, operation: operation}} + end) + end + defp map_color(:transitioned), do: "blue" defp map_color(:pending), do: "green" defp map_color(:download_blobs), do: "yellow" diff --git a/lib/lambda_ethereum_consensus/state_transition/operations.ex b/lib/lambda_ethereum_consensus/state_transition/operations.ex index 7a2e46bcd..76acd20e9 100644 --- a/lib/lambda_ethereum_consensus/state_transition/operations.ex +++ b/lib/lambda_ethereum_consensus/state_transition/operations.ex @@ -3,6 +3,7 @@ defmodule LambdaEthereumConsensus.StateTransition.Operations do This module contains functions for handling state transition """ + alias LambdaEthereumConsensus.Metrics alias LambdaEthereumConsensus.StateTransition.Accessors alias LambdaEthereumConsensus.StateTransition.Math alias LambdaEthereumConsensus.StateTransition.Misc @@ -919,19 +920,31 @@ defmodule LambdaEthereumConsensus.StateTransition.Operations do # Ensure that outstanding deposits are processed up to the maximum number of deposits with :ok <- verify_deposits(state, body) do {:ok, state} - |> for_ops(body.proposer_slashings, &process_proposer_slashing/2) - |> for_ops(body.attester_slashings, &process_attester_slashing/2) - |> Utils.map_ok(&process_attestation_batch(&1, body.attestations)) - |> for_ops(body.deposits, &process_deposit/2) - |> for_ops(body.voluntary_exits, &process_voluntary_exit/2) - |> for_ops(body.bls_to_execution_changes, &process_bls_to_execution_change/2) + |> for_ops(:proposer_slashing, body.proposer_slashings, &process_proposer_slashing/2) + |> for_ops(:attester_slashing, body.attester_slashings, &process_attester_slashing/2) + |> apply_op(:attestation_batch, &process_attestation_batch(&1, body.attestations)) + |> for_ops(:deposit, body.deposits, &process_deposit/2) + |> for_ops(:voluntary_exit, body.voluntary_exits, &process_voluntary_exit/2) + |> for_ops( + :bls_to_execution_change, + body.bls_to_execution_changes, + &process_bls_to_execution_change/2 + ) end end - defp for_ops(acc, operations, func) do - Enum.reduce_while(operations, acc, fn - operation, {:ok, state} -> {:cont, func.(state, operation)} - _, {:error, reason} -> {:halt, {:error, reason}} + defp apply_op(acc, op_name, func) do + Metrics.span_operation(:on_block, :process_block_operations, op_name, fn -> + Utils.map_ok(acc, func) + end) + end + + defp for_ops(acc, op_name, operations, func) do + Metrics.span_operation(:on_block, :process_block_operations, op_name, fn -> + Enum.reduce_while(operations, acc, fn + operation, {:ok, state} -> {:cont, func.(state, operation)} + _, {:error, reason} -> {:halt, {:error, reason}} + end) end) end diff --git a/lib/lambda_ethereum_consensus/state_transition/state_transition.ex b/lib/lambda_ethereum_consensus/state_transition/state_transition.ex index 180769415..486bbb190 100644 --- a/lib/lambda_ethereum_consensus/state_transition/state_transition.ex +++ b/lib/lambda_ethereum_consensus/state_transition/state_transition.ex @@ -4,6 +4,7 @@ defmodule LambdaEthereumConsensus.StateTransition do """ require Logger + alias LambdaEthereumConsensus.Metrics alias LambdaEthereumConsensus.StateTransition.Accessors alias LambdaEthereumConsensus.StateTransition.EpochProcessing alias LambdaEthereumConsensus.StateTransition.Misc @@ -108,17 +109,23 @@ defmodule LambdaEthereumConsensus.StateTransition do state |> EpochProcessing.process_justification_and_finalization() - |> map_ok(&EpochProcessing.process_inactivity_updates/1) - |> map_ok(&EpochProcessing.process_rewards_and_penalties/1) - |> map_ok(&EpochProcessing.process_registry_updates/1) - |> map_ok(&EpochProcessing.process_slashings/1) - |> map_ok(&EpochProcessing.process_eth1_data_reset/1) - |> map_ok(&EpochProcessing.process_effective_balance_updates/1) - |> map_ok(&EpochProcessing.process_slashings_reset/1) - |> map_ok(&EpochProcessing.process_randao_mixes_reset/1) - |> map_ok(&EpochProcessing.process_historical_summaries_update/1) - |> map_ok(&EpochProcessing.process_participation_flag_updates/1) - |> map_ok(&EpochProcessing.process_sync_committee_updates/1) + |> epoch_op(:inactivity_updates, &EpochProcessing.process_inactivity_updates/1) + |> epoch_op(:rewards_and_penalties, &EpochProcessing.process_rewards_and_penalties/1) + |> epoch_op(:registry_updates, &EpochProcessing.process_registry_updates/1) + |> epoch_op(:slashings, &EpochProcessing.process_slashings/1) + |> epoch_op(:eth1_data_reset, &EpochProcessing.process_eth1_data_reset/1) + |> epoch_op(:effective_balance_updates, &EpochProcessing.process_effective_balance_updates/1) + |> epoch_op(:slashings_reset, &EpochProcessing.process_slashings_reset/1) + |> epoch_op(:randao_mixes_reset, &EpochProcessing.process_randao_mixes_reset/1) + |> epoch_op( + :historical_summaries_update, + &EpochProcessing.process_historical_summaries_update/1 + ) + |> epoch_op( + :participation_flag_updates, + &EpochProcessing.process_participation_flag_updates/1 + ) + |> epoch_op(:sync_committee_updates, &EpochProcessing.process_sync_committee_updates/1) |> tap(fn _ -> end_time = System.monotonic_time(:millisecond) Logger.debug("[Epoch processing] took #{end_time - start_time} ms") @@ -136,16 +143,26 @@ defmodule LambdaEthereumConsensus.StateTransition do start_time = System.monotonic_time(:millisecond) {:ok, state} - |> map_ok(&Operations.process_block_header(&1, block)) - |> map_ok(&Operations.process_withdrawals(&1, block.body.execution_payload)) - |> map_ok(&Operations.process_execution_payload(&1, block.body)) - |> map_ok(&Operations.process_randao(&1, block.body)) - |> map_ok(&Operations.process_eth1_data(&1, block.body)) + |> block_op(:block_header, &Operations.process_block_header(&1, block)) + |> block_op(:withdrawals, &Operations.process_withdrawals(&1, block.body.execution_payload)) + |> block_op(:execution_payload, &Operations.process_execution_payload(&1, block.body)) + |> block_op(:randao, &Operations.process_randao(&1, block.body)) + |> block_op(:eth1_data, &Operations.process_eth1_data(&1, block.body)) |> map_ok(&Operations.process_operations(&1, block.body)) - |> map_ok(&Operations.process_sync_aggregate(&1, block.body.sync_aggregate)) + |> block_op( + :sync_aggregate, + &Operations.process_sync_aggregate(&1, block.body.sync_aggregate) + ) |> tap(fn _ -> end_time = System.monotonic_time(:millisecond) Logger.debug("[Block processing] took #{end_time - start_time} ms") end) end + + def block_op(state, operation, f), do: apply_op(state, :process_block, operation, f) + def epoch_op(state, operation, f), do: apply_op(state, :epoch, operation, f) + + def apply_op(state, transition, operation, f) do + Metrics.span_operation(:on_block, transition, operation, fn -> map_ok(state, f) end) + end end diff --git a/lib/lambda_ethereum_consensus/telemetry.ex b/lib/lambda_ethereum_consensus/telemetry.ex index af38fa691..cdbe1921c 100644 --- a/lib/lambda_ethereum_consensus/telemetry.ex +++ b/lib/lambda_ethereum_consensus/telemetry.ex @@ -140,6 +140,10 @@ defmodule LambdaEthereumConsensus.Telemetry do tags: [:module, :action] ), counter("db.latency.stop.count", unit: {:native, :millisecond}, tags: [:module, :action]), + last_value("fork_choice.latency.stop.duration", + unit: {:native, :millisecond}, + tags: [:handler, :transition, :operation] + ), # ForkChoice Metrics last_value("fork_choice.recompute_head.stop.duration", From faacc9b6b60f164dd9c16bc52556b6a174ff7f91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Arjovsky?= Date: Fri, 19 Jul 2024 12:50:12 +0200 Subject: [PATCH 2/2] add grafana panel --- .../grafana/provisioning/dashboards/home.json | 741 ++++++++++-------- 1 file changed, 417 insertions(+), 324 deletions(-) diff --git a/metrics/grafana/provisioning/dashboards/home.json b/metrics/grafana/provisioning/dashboards/home.json index b4156322f..2b628c33d 100644 --- a/metrics/grafana/provisioning/dashboards/home.json +++ b/metrics/grafana/provisioning/dashboards/home.json @@ -28,89 +28,6 @@ "links": [], "liveNow": false, "panels": [ - { - "datasource": { - "uid": "PBFA97CFB590B2093", - "type": "prometheus" - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 0 - }, - "id": 31, - "options": { - "nodes": {}, - "edges": {} - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "blocks_status_total", - "format": "table", - "fullMetaSearch": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "A", - "useBackend": false, - "exemplar": false - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "blocks_relationship_total", - "format": "table", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "__auto", - "range": false, - "refId": "B", - "useBackend": false, - "exemplar": false - } - ], - "title": "Blockchain View", - "transformations": [ - { - "id": "filterByValue", - "options": { - "filters": [ - { - "fieldName": "Value #A", - "config": { - "id": "equal", - "options": { - "value": 0 - } - } - } - ], - "type": "exclude", - "match": "any" - }, - "filter": { - "id": "byRefId", - "options": "A" - }, - "topic": "series" - } - ], - "type": "nodeGraph" - }, { "datasource": { "type": "prometheus", @@ -119,49 +36,51 @@ "fieldConfig": { "defaults": { "color": { - "mode": "thresholds" + "mode": "palette-classic" }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } }, + "mappings": [], "unit": "ms" }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 12, "w": 12, "x": 0, "y": 0 }, - "id": 30, + "id": 32, "options": { - "displayMode": "gradient", - "minVizHeight": 10, - "minVizWidth": 0, - "orientation": "horizontal", + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "donut", "reduceOptions": { "calcs": [ - "mean" + "lastNotNull" ], "fields": "", "values": false }, - "showUnfilled": true, - "valueMode": "color" + "tooltip": { + "mode": "single", + "sort": "none" + } }, - "pluginVersion": "9.5.1", "targets": [ { "datasource": { @@ -169,14 +88,38 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "builder", - "expr": "db_latency_stop_duration", - "legendFormat": "{{module}} - {{action}}", + "expr": "fork_choice_latency_stop_duration{handler=\"on_block\"}", + "legendFormat": "{{handler}} - {{transition}} - {{operation}}", "range": true, - "refId": "Mean time per db operation" + "refId": "on_block" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "fork_choice_recompute_head_stop_duration", + "hide": false, + "legendFormat": "recompute_head", + "range": true, + "refId": "recompute head" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "fork_choice_latency_stop_duration{handler!=\"on_block\"}", + "hide": false, + "legendFormat": "{{handler}}", + "range": true, + "refId": "Other handlers" } ], - "title": "Mean time per db operation", - "type": "bargauge" + "title": "Fork choice handler times", + "type": "piechart" }, { "datasource": { @@ -312,77 +255,16 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "Bytes", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 6, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "decbytes" - }, - "overrides": [] - }, "gridPos": { "h": 8, "w": 12, "x": 0, - "y": 8 + "y": 12 }, - "id": 1, + "id": 31, "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "edges": {}, + "nodes": {} }, "targets": [ { @@ -392,18 +274,64 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "db_size_total", + "exemplar": false, + "expr": "blocks_status_total", + "format": "table", "fullMetaSearch": false, "includeNullMetadata": true, "instant": true, "legendFormat": "__auto", - "range": true, + "range": false, "refId": "A", "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "blocks_relationship_total", + "format": "table", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "B", + "useBackend": false } ], - "title": "Database size", - "type": "timeseries" + "title": "Blockchain View", + "transformations": [ + { + "filter": { + "id": "byRefId", + "options": "A" + }, + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "equal", + "options": { + "value": 0 + } + }, + "fieldName": "Value #A" + } + ], + "match": "any", + "type": "exclude" + }, + "topic": "series" + } + ], + "type": "nodeGraph" }, { "datasource": { @@ -516,8 +444,166 @@ "useBackend": false } ], - "title": "P2P Requests", - "type": "timeseries" + "title": "P2P Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 14, + "maxDataPoints": 25, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "Oranges", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "show": true, + "showLegend": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "show": true, + "sort": "none", + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false + } + }, + "pluginVersion": "9.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "network_pubsub_topic_active_active", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "{{topic}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "PubSub topics joined", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 15, + "w": 12, + "x": 0, + "y": 20 + }, + "id": 30, + "options": { + "displayMode": "gradient", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "valueMode": "color" + }, + "pluginVersion": "9.5.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "db_latency_stop_duration", + "legendFormat": "{{module}} - {{action}}", + "range": true, + "refId": "Mean time per db operation" + } + ], + "title": "Mean time per db operation", + "type": "bargauge" }, { "datasource": { @@ -544,8 +630,8 @@ "viz": false }, "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 1, + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" }, @@ -572,27 +658,25 @@ "value": 80 } ] - }, - "unit": "ms" + } }, "overrides": [] }, "gridPos": { - "h": 8, + "h": 6, "w": 12, - "x": 0, - "y": 16 + "x": 12, + "y": 24 }, - "id": 27, + "id": 11, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": false + "showLegend": true }, "tooltip": { - "maxHeight": 600, "mode": "single", "sort": "none" } @@ -603,20 +687,28 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "fork_choice_recompute_head_stop_duration", - "fullMetaSearch": false, + "editorMode": "code", + "expr": "rate(peers_challenge_count{result=\"failed\"}[$__rate_interval])", + "instant": false, + "legendFormat": "failed", + "range": true, + "refId": "Challenge failed" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(peers_challenge_count{result=\"passed\"}[$__rate_interval])", "hide": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "__auto", + "instant": false, + "legendFormat": "passed", "range": true, - "refId": "A", - "useBackend": false + "refId": "Challenge passed" } ], - "title": "ForkChoice RecomputeHead Duration", + "title": "Peer challenges", "type": "timeseries" }, { @@ -626,15 +718,52 @@ }, "fieldConfig": { "defaults": { + "color": { + "mode": "palette-classic" + }, "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", "hideFrom": { "legend": false, "tooltip": false, "viz": false }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, "scaleDistribution": { "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] } }, "overrides": [] @@ -643,50 +772,21 @@ "h": 6, "w": 12, "x": 12, - "y": 18 + "y": 30 }, - "id": 14, - "maxDataPoints": 25, + "id": 15, "options": { - "calculate": false, - "cellGap": 1, - "color": { - "exponent": 0.5, - "fill": "dark-orange", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Oranges", - "steps": 64 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "show": true, "showLegend": true }, - "rowsFrame": { - "layout": "auto" - }, "tooltip": { "mode": "single", - "show": true, - "sort": "none", - "yHistogram": false - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false + "sort": "none" } }, - "pluginVersion": "9.5.1", "targets": [ { "datasource": { @@ -696,7 +796,7 @@ "disableTextWrap": false, "editorMode": "code", "exemplar": false, - "expr": "network_pubsub_topic_active_active", + "expr": "network_pubsub_topics_graft_count{} - network_pubsub_topics_prune_count{}", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -707,8 +807,8 @@ "useBackend": false } ], - "title": "PubSub topics joined", - "type": "heatmap" + "title": "Grafted", + "type": "timeseries" }, { "datasource": { @@ -723,7 +823,7 @@ "custom": { "axisCenteredZero": false, "axisColorMode": "text", - "axisLabel": "", + "axisLabel": "Bytes", "axisPlacement": "auto", "barAlignment": 0, "drawStyle": "line", @@ -735,8 +835,8 @@ "viz": false }, "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 1, + "lineWidth": 1, + "pointSize": 6, "scaleDistribution": { "type": "linear" }, @@ -764,7 +864,7 @@ } ] }, - "unit": "ms" + "unit": "decbytes" }, "overrides": [] }, @@ -772,9 +872,9 @@ "h": 8, "w": 12, "x": 0, - "y": 24 + "y": 35 }, - "id": 25, + "id": 1, "options": { "legend": { "calcs": [], @@ -783,7 +883,6 @@ "showLegend": false }, "tooltip": { - "maxHeight": 600, "mode": "single", "sort": "none" } @@ -796,9 +895,8 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "db_latency_stop_duration{module=\"fork_choice\", action=\"persist\"}", + "expr": "db_size_total", "fullMetaSearch": false, - "hide": false, "includeNullMetadata": true, "instant": true, "legendFormat": "__auto", @@ -807,7 +905,7 @@ "useBackend": false } ], - "title": "ForkChoice Persisting Store Duration", + "title": "Database size", "type": "timeseries" }, { @@ -871,9 +969,9 @@ "h": 6, "w": 12, "x": 12, - "y": 24 + "y": 36 }, - "id": 11, + "id": 16, "options": { "legend": { "calcs": [], @@ -892,28 +990,21 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "disableTextWrap": false, "editorMode": "code", - "expr": "rate(peers_challenge_count{result=\"failed\"}[$__rate_interval])", - "instant": false, - "legendFormat": "failed", - "range": true, - "refId": "Challenge failed" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "rate(peers_challenge_count{result=\"passed\"}[$__rate_interval])", - "hide": false, + "exemplar": false, + "expr": "rate(network_pubsub_topics_deliver_message_count{}[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, "instant": false, - "legendFormat": "passed", + "interval": "", + "legendFormat": "{{topic}}", "range": true, - "refId": "Challenge passed" + "refId": "A", + "useBackend": false } ], - "title": "Peer challenges", + "title": "Deliver Messages", "type": "timeseries" }, { @@ -977,9 +1068,9 @@ "h": 6, "w": 12, "x": 12, - "y": 30 + "y": 42 }, - "id": 15, + "id": 19, "options": { "legend": { "calcs": [], @@ -1001,7 +1092,7 @@ "disableTextWrap": false, "editorMode": "code", "exemplar": false, - "expr": "network_pubsub_topics_graft_count{} - network_pubsub_topics_prune_count{}", + "expr": "rate(network_pubsub_topics_un_deliverable_message_count{}[$__rate_interval])", "fullMetaSearch": false, "includeNullMetadata": true, "instant": false, @@ -1012,7 +1103,7 @@ "useBackend": false } ], - "title": "Grafted", + "title": "Undeliverable Messages", "type": "timeseries" }, { @@ -1077,9 +1168,9 @@ "h": 8, "w": 12, "x": 0, - "y": 32 + "y": 43 }, - "id": 28, + "id": 27, "options": { "legend": { "calcs": [], @@ -1101,7 +1192,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "db_latency_stop_duration{module=\"subnet\", action=\"persist\"}", + "expr": "fork_choice_recompute_head_stop_duration", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -1112,7 +1203,7 @@ "useBackend": false } ], - "title": "SubnetInfo Persisting Duration", + "title": "ForkChoice RecomputeHead Duration", "type": "timeseries" }, { @@ -1140,8 +1231,8 @@ "viz": false }, "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, + "lineWidth": 2, + "pointSize": 1, "scaleDistribution": { "type": "linear" }, @@ -1168,25 +1259,27 @@ "value": 80 } ] - } + }, + "unit": "ms" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, - "x": 12, - "y": 36 + "x": 0, + "y": 51 }, - "id": 16, + "id": 25, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" } @@ -1198,20 +1291,19 @@ "uid": "PBFA97CFB590B2093" }, "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "rate(network_pubsub_topics_deliver_message_count{}[$__rate_interval])", + "editorMode": "builder", + "expr": "db_latency_stop_duration{module=\"fork_choice\", action=\"persist\"}", "fullMetaSearch": false, + "hide": false, "includeNullMetadata": true, - "instant": false, - "interval": "", - "legendFormat": "{{topic}}", + "instant": true, + "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false } ], - "title": "Deliver Messages", + "title": "ForkChoice Persisting Store Duration", "type": "timeseries" }, { @@ -1276,9 +1368,9 @@ "h": 8, "w": 12, "x": 0, - "y": 40 + "y": 59 }, - "id": 29, + "id": 28, "options": { "legend": { "calcs": [], @@ -1300,7 +1392,7 @@ }, "disableTextWrap": false, "editorMode": "builder", - "expr": "db_latency_stop_duration{module=\"subnet\", action=\"fetch\"}", + "expr": "db_latency_stop_duration{module=\"subnet\", action=\"persist\"}", "fullMetaSearch": false, "hide": false, "includeNullMetadata": true, @@ -1311,7 +1403,7 @@ "useBackend": false } ], - "title": "SubnetInfo Fetching Duration", + "title": "SubnetInfo Persisting Duration", "type": "timeseries" }, { @@ -1339,8 +1431,8 @@ "viz": false }, "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, + "lineWidth": 2, + "pointSize": 1, "scaleDistribution": { "type": "linear" }, @@ -1367,25 +1459,27 @@ "value": 80 } ] - } + }, + "unit": "ms" }, "overrides": [] }, "gridPos": { - "h": 6, + "h": 8, "w": 12, - "x": 12, - "y": 42 + "x": 0, + "y": 67 }, - "id": 19, + "id": 29, "options": { "legend": { "calcs": [], "displayMode": "list", "placement": "bottom", - "showLegend": true + "showLegend": false }, "tooltip": { + "maxHeight": 600, "mode": "single", "sort": "none" } @@ -1397,20 +1491,19 @@ "uid": "PBFA97CFB590B2093" }, "disableTextWrap": false, - "editorMode": "code", - "exemplar": false, - "expr": "rate(network_pubsub_topics_un_deliverable_message_count{}[$__rate_interval])", + "editorMode": "builder", + "expr": "db_latency_stop_duration{module=\"subnet\", action=\"fetch\"}", "fullMetaSearch": false, + "hide": false, "includeNullMetadata": true, - "instant": false, - "interval": "", - "legendFormat": "{{topic}}", + "instant": true, + "legendFormat": "__auto", "range": true, "refId": "A", "useBackend": false } ], - "title": "Undeliverable Messages", + "title": "SubnetInfo Fetching Duration", "type": "timeseries" }, { @@ -1475,7 +1568,7 @@ "h": 8, "w": 12, "x": 0, - "y": 48 + "y": 75 }, "id": 26, "options": { @@ -1555,7 +1648,7 @@ "h": 6, "w": 12, "x": 0, - "y": 56 + "y": 83 }, "hideTimeOverride": true, "id": 24, @@ -1651,7 +1744,7 @@ "h": 6, "w": 12, "x": 0, - "y": 62 + "y": 89 }, "id": 6, "options": { @@ -1760,7 +1853,7 @@ "h": 6, "w": 24, "x": 0, - "y": 68 + "y": 95 }, "id": 2, "options": { @@ -1925,7 +2018,7 @@ "h": 6, "w": 12, "x": 0, - "y": 74 + "y": 101 }, "id": 3, "options": { @@ -2036,7 +2129,7 @@ "h": 6, "w": 12, "x": 12, - "y": 74 + "y": 101 }, "id": 12, "options": { @@ -2144,7 +2237,7 @@ "h": 6, "w": 5, "x": 0, - "y": 80 + "y": 107 }, "id": 7, "options": { @@ -2204,7 +2297,7 @@ "h": 6, "w": 7, "x": 5, - "y": 80 + "y": 107 }, "id": 5, "options": { @@ -2304,7 +2397,7 @@ "h": 6, "w": 12, "x": 0, - "y": 86 + "y": 113 }, "id": 13, "options": { @@ -2403,7 +2496,7 @@ "h": 6, "w": 12, "x": 0, - "y": 92 + "y": 119 }, "id": 17, "options": { @@ -2502,7 +2595,7 @@ "h": 6, "w": 12, "x": 0, - "y": 98 + "y": 125 }, "id": 21, "options": { @@ -2626,7 +2719,7 @@ "h": 6, "w": 12, "x": 0, - "y": 104 + "y": 131 }, "id": 20, "options": { @@ -2725,7 +2818,7 @@ "h": 6, "w": 12, "x": 0, - "y": 110 + "y": 137 }, "id": 18, "options": { @@ -2779,6 +2872,6 @@ "timezone": "", "title": "Node", "uid": "90EXFQnIk", - "version": 3, + "version": 21, "weekStart": "" -} \ No newline at end of file +}