diff --git a/lib/lambda_ethereum_consensus/metrics.ex b/lib/lambda_ethereum_consensus/metrics.ex index f4d303463..b46388925 100644 --- a/lib/lambda_ethereum_consensus/metrics.ex +++ b/lib/lambda_ethereum_consensus/metrics.ex @@ -88,22 +88,6 @@ defmodule LambdaEthereumConsensus.Metrics do block_status_execute(root, new_status, slot, 1) end - defp block_status_execute(root, status, slot, value) do - hex_root = Base.encode16(root) - - Logger.debug( - "[Metrics] slot = #{inspect(slot)}, status = #{inspect(status)}, value = #{inspect(value)}" - ) - - :telemetry.execute([:blocks, :status], %{total: value}, %{ - id: hex_root, - mainstat: status, - color: map_color(status), - title: slot, - detail__root: hex_root - }) - end - def block_relationship(nil, _), do: :ok def block_relationship(parent_root, root) do @@ -126,6 +110,28 @@ defmodule LambdaEthereumConsensus.Metrics do end) end + def handler_span(module, action, f) do + :telemetry.span([:libp2pport, :handler], %{}, fn -> + {f.(), %{module: module, action: action}} + end) + end + + defp block_status_execute(root, status, slot, value) do + hex_root = Base.encode16(root) + + Logger.debug( + "[Metrics] slot = #{inspect(slot)}, status = #{inspect(status)}, value = #{inspect(value)}" + ) + + :telemetry.execute([:blocks, :status], %{total: value}, %{ + id: hex_root, + mainstat: status, + color: map_color(status), + title: slot, + detail__root: hex_root + }) + end + defp map_color(:transitioned), do: "blue" defp map_color(:pending), do: "green" defp map_color(:download_blobs), do: "yellow" diff --git a/lib/lambda_ethereum_consensus/p2p/blob_downloader.ex b/lib/lambda_ethereum_consensus/p2p/blob_downloader.ex index 6888ba9aa..558855a86 100644 --- a/lib/lambda_ethereum_consensus/p2p/blob_downloader.ex +++ b/lib/lambda_ethereum_consensus/p2p/blob_downloader.ex @@ -5,6 +5,7 @@ defmodule LambdaEthereumConsensus.P2P.BlobDownloader do require Logger alias LambdaEthereumConsensus.Libp2pPort + alias LambdaEthereumConsensus.Metrics alias LambdaEthereumConsensus.P2P alias LambdaEthereumConsensus.P2P.ReqResp alias Types.BlobSidecar @@ -38,7 +39,13 @@ defmodule LambdaEthereumConsensus.P2P.BlobDownloader do |> ReqResp.encode_request() Libp2pPort.send_async_request(peer_id, @blobs_by_range_protocol_id, request, fn response -> - handle_blobs_by_range_response(response, peer_id, count, slot, retries, on_blobs) + Metrics.handler_span( + "response_handler", + "blob_sidecars_by_range", + fn -> + handle_blobs_by_range_response(response, peer_id, count, slot, retries, on_blobs) + end + ) end) end @@ -85,7 +92,11 @@ defmodule LambdaEthereumConsensus.P2P.BlobDownloader do request = ReqResp.encode_request({identifiers, TypeAliases.blob_sidecars_by_root_request()}) Libp2pPort.send_async_request(peer_id, @blobs_by_root_protocol_id, request, fn response -> - handle_blobs_by_root(response, peer_id, identifiers, retries, on_blobs) + Metrics.handler_span( + "response_handler", + "blob_sidecars_by_root", + fn -> handle_blobs_by_root(response, peer_id, identifiers, retries, on_blobs) end + ) end) end diff --git a/lib/lambda_ethereum_consensus/p2p/block_downloader.ex b/lib/lambda_ethereum_consensus/p2p/block_downloader.ex index 9278ee283..06e97fa9b 100644 --- a/lib/lambda_ethereum_consensus/p2p/block_downloader.ex +++ b/lib/lambda_ethereum_consensus/p2p/block_downloader.ex @@ -5,6 +5,7 @@ defmodule LambdaEthereumConsensus.P2P.BlockDownloader do require Logger alias LambdaEthereumConsensus.Libp2pPort + alias LambdaEthereumConsensus.Metrics alias LambdaEthereumConsensus.P2P alias LambdaEthereumConsensus.P2P.ReqResp alias Types.SignedBeaconBlock @@ -65,7 +66,13 @@ defmodule LambdaEthereumConsensus.P2P.BlockDownloader do |> ReqResp.encode_request() Libp2pPort.send_async_request(peer_id, @blocks_by_range_protocol_id, request, fn response -> - handle_blocks_by_range_response(response, slot, count, retries, peer_id, on_blocks) + Metrics.handler_span( + "response_handler", + "blocks_by_range", + fn -> + handle_blocks_by_range_response(response, slot, count, retries, peer_id, on_blocks) + end + ) end) end @@ -127,7 +134,11 @@ defmodule LambdaEthereumConsensus.P2P.BlockDownloader do request = ReqResp.encode_request({roots, TypeAliases.beacon_blocks_by_root_request()}) Libp2pPort.send_async_request(peer_id, @blocks_by_root_protocol_id, request, fn response -> - handle_blocks_by_root_response(response, roots, on_blocks, peer_id, retries) + Metrics.handler_span( + "response_handler", + "blocks_by_root", + fn -> handle_blocks_by_root_response(response, roots, on_blocks, peer_id, retries) end + ) end) end diff --git a/lib/lambda_ethereum_consensus/p2p/incoming_requests_handler.ex b/lib/lambda_ethereum_consensus/p2p/incoming_requests_handler.ex index 6e342073b..6c235d812 100644 --- a/lib/lambda_ethereum_consensus/p2p/incoming_requests_handler.ex +++ b/lib/lambda_ethereum_consensus/p2p/incoming_requests_handler.ex @@ -4,6 +4,7 @@ defmodule LambdaEthereumConsensus.P2P.IncomingRequestsHandler do """ alias LambdaEthereumConsensus.ForkChoice + alias LambdaEthereumConsensus.Metrics alias LambdaEthereumConsensus.P2P.Metadata alias LambdaEthereumConsensus.P2P.ReqResp alias LambdaEthereumConsensus.Store.BlockDb @@ -31,8 +32,8 @@ defmodule LambdaEthereumConsensus.P2P.IncomingRequestsHandler do Logger.debug("'#{name}' request received") result = - :telemetry.span([:port, :request], %{}, fn -> - {handle_req(name, message_id, message), %{module: "handler", request: inspect(name)}} + Metrics.handler_span("request_handler", name |> String.split("/") |> List.first(), fn -> + handle_req(name, message_id, message) end) case result do diff --git a/lib/lambda_ethereum_consensus/telemetry.ex b/lib/lambda_ethereum_consensus/telemetry.ex index cdbe1921c..a4805e328 100644 --- a/lib/lambda_ethereum_consensus/telemetry.ex +++ b/lib/lambda_ethereum_consensus/telemetry.ex @@ -85,14 +85,6 @@ defmodule LambdaEthereumConsensus.Telemetry do counter("network.pubsub_topics_un_deliverable_message.count", tags: [:topic]), counter("network.pubsub_topics_validate_message.count", tags: [:topic]), counter("port.message.count", tags: [:function, :direction]), - last_value("port.request.stop.duration", - unit: {:native, :millisecond}, - tags: [:module, :request] - ), - last_value("port.request.exception.duration", - unit: {:native, :millisecond}, - tags: [:module, :request] - ), sum("network.request.blocks", tags: [:result, :type, :reason]), # Sync metrics @@ -140,6 +132,18 @@ defmodule LambdaEthereumConsensus.Telemetry do tags: [:module, :action] ), counter("db.latency.stop.count", unit: {:native, :millisecond}, tags: [:module, :action]), + last_value("libp2pport.handler.stop.duration", + unit: {:native, :millisecond}, + tags: [:module, :action] + ), + last_value("libp2pport.handler.exception.duration", + unit: {:native, :millisecond}, + tags: [:module, :action] + ), + counter("libp2pport.handler.stop.count", + unit: {:native, :millisecond}, + tags: [:module, :action] + ), last_value("fork_choice.latency.stop.duration", unit: {:native, :millisecond}, tags: [:handler, :transition, :operation] diff --git a/lib/libp2p_port.ex b/lib/libp2p_port.ex index d3dc898f4..b7d462d56 100644 --- a/lib/libp2p_port.ex +++ b/lib/libp2p_port.ex @@ -488,8 +488,13 @@ defmodule LambdaEthereumConsensus.Libp2pPort do }) case Map.fetch(subscribers, gs.topic) do - {:ok, module} -> module.handle_gossip_message(gs.topic, gs.msg_id, gs.message) - :error -> Logger.error("[Gossip] Received gossip from unknown topic: #{gs.topic}.") + {:ok, module} -> + Metrics.handler_span("gossip_handler", gs.topic, fn -> + module.handle_gossip_message(gs.topic, gs.msg_id, gs.message) + end) + + :error -> + Logger.error("[Gossip] Received gossip from unknown topic: #{gs.topic}.") end state diff --git a/metrics/grafana/provisioning/dashboards/home.json b/metrics/grafana/provisioning/dashboards/home.json index 2b628c33d..e72e9d732 100644 --- a/metrics/grafana/provisioning/dashboards/home.json +++ b/metrics/grafana/provisioning/dashboards/home.json @@ -28,6 +28,156 @@ "links": [], "liveNow": false, "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "color": { + "mode": "palette-classic" + }, + "mappings": [] + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 31, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "pieType": "pie", + "tooltip": { + "mode": "single", + "sort": "none", + "maxHeight": 600 + }, + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "right", + "values": [ + "value", + "percent" + ] + }, + "displayLabels": [] + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "libp2pport_handler_stop_count", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{module}} - {{action}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Libp2pPort Handlers Count", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "color": { + "mode": "palette-classic" + }, + "mappings": [], + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 30, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "mean" + ], + "fields": "" + }, + "pieType": "donut", + "tooltip": { + "mode": "single", + "sort": "none", + "maxHeight": 600 + }, + "legend": { + "showLegend": true, + "displayMode": "table", + "placement": "right", + "values": [ + "value", + "percent" + ] + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": false, + "expr": "libp2pport_handler_stop_duration", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "{{module}} - {{action}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Libp2pPort Handlers Time", + "type": "piechart" + }, { "datasource": { "type": "prometheus", @@ -56,7 +206,7 @@ "x": 0, "y": 0 }, - "id": 32, + "id": 29, "options": { "displayLabels": [], "legend": { @@ -261,7 +411,7 @@ "x": 0, "y": 12 }, - "id": 31, + "id": 28, "options": { "edges": {}, "nodes": {} @@ -572,7 +722,7 @@ "x": 0, "y": 20 }, - "id": 30, + "id": 27, "options": { "displayMode": "gradient", "minVizHeight": 10, @@ -1106,206 +1256,6 @@ "title": "Undeliverable Messages", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 1, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 43 - }, - "id": 27, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "fork_choice_recompute_head_stop_duration", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "ForkChoice RecomputeHead Duration", - "type": "timeseries" - }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 1, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 51 - }, - "id": 25, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "db_latency_stop_duration{module=\"fork_choice\", action=\"persist\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "ForkChoice Persisting Store Duration", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -1370,7 +1320,7 @@ "x": 0, "y": 59 }, - "id": 28, + "id": 26, "options": { "legend": { "calcs": [], @@ -1470,7 +1420,7 @@ "x": 0, "y": 67 }, - "id": 29, + "id": 25, "options": { "legend": { "calcs": [], @@ -1506,106 +1456,6 @@ "title": "SubnetInfo Fetching Duration", "type": "timeseries" }, - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "lineInterpolation": "linear", - "lineWidth": 2, - "pointSize": 1, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "ms" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 75 - }, - "id": 26, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": false - }, - "tooltip": { - "maxHeight": 600, - "mode": "single", - "sort": "none" - } - }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "disableTextWrap": false, - "editorMode": "builder", - "expr": "db_latency_stop_duration{module=\"fork_choice\", action=\"fetch\"}", - "fullMetaSearch": false, - "hide": false, - "includeNullMetadata": true, - "instant": true, - "legendFormat": "__auto", - "range": true, - "refId": "A", - "useBackend": false - } - ], - "title": "ForkChoice Fetching Store Duration", - "type": "timeseries" - }, { "datasource": { "type": "prometheus", @@ -2874,4 +2724,4 @@ "uid": "90EXFQnIk", "version": 21, "weekStart": "" -} +} \ No newline at end of file