feat(cl-mimicry): Add Gossipsub_beacon_block events

Makefile

@@ -3,4 +3,6 @@ proto:
  protoc --proto_path=./ --go_opt=module=github.com/ethpandaops/xatu/pkg/proto/eth/v2 --go_out=./pkg/proto/eth/v2/ pkg/proto/eth/v2/*.proto
  protoc --proto_path=./ --proto_path=./pkg/proto/eth/v1 --proto_path=./pkg/proto/eth/v2 --go_opt=module=github.com/ethpandaops/xatu/pkg/proto/xatu --go-grpc_out=. --go-grpc_opt=paths=source_relative --go_out=./pkg/proto/xatu pkg/proto/xatu/*.proto
  protoc --proto_path=./ --go_opt=module=github.com/ethpandaops/xatu/pkg/proto/blockprint --go_out=./pkg/proto/blockprint pkg/proto/blockprint/*.proto
- protoc --proto_path=./ --go_opt=module=github.com/ethpandaops/xatu/pkg/proto/libp2p --go_out=./pkg/proto/libp2p pkg/proto/libp2p/*.proto
+ protoc --proto_path=./ --go_opt=module=github.com/ethpandaops/xatu/pkg/proto/libp2p --go_out=./pkg/proto/libp2p pkg/proto/libp2p/*.proto
+ protoc --proto_path=./ --go_opt=module=github.com/ethpandaops/xatu/pkg/proto/libp2p/gossipsub --go_out=./pkg/proto/libp2p/gossipsub pkg/proto/libp2p/gossipsub/*.proto
+

deploy/local/docker-compose/vector-http-kafka.yaml

@@ -63,6 +63,7 @@ transforms:
  libp2p_trace_join: .event.name == "LIBP2P_TRACE_JOIN"
  libp2p_trace_handle_metadata: .event.name == "LIBP2P_TRACE_HANDLE_METADATA"
  libp2p_trace_handle_status: .event.name == "LIBP2P_TRACE_HANDLE_STATUS"
+ libp2p_trace_gossipsub_beacon_block : .event.name == "LIBP2P_TRACE_GOSSIPSUB_BEACON_BLOCK"
 sinks:
  metrics:
  type: prometheus_exporter
@@ -822,3 +823,19 @@ sinks:
  enabled: true
  encoding:
  codec: json
+ libp2p_trace_gossipsub_beacon_block_kafka:
+ type: kafka
+ buffer:
+ max_events: 500000
+ batch:
+ timeout_secs: 0.5
+ inputs:
+ - xatu_server_events_router.libp2p_trace_gossipsub_beacon_block
+ bootstrap_servers: "${KAFKA_BROKERS}"
+ key_field: "event.id"
+ topic: libp2p-trace-gossipsub-beacon-block
+ compression: snappy
+ healthcheck:
+ enabled: true
+ encoding:
+ codec: json

deploy/local/docker-compose/vector-kafka-clickhouse.yaml

@@ -359,6 +359,7 @@ transforms:
  libp2p_trace_join: .event.name == "LIBP2P_TRACE_JOIN"
  libp2p_trace_handle_metadata: .event.name == "LIBP2P_TRACE_HANDLE_METADATA"
  libp2p_trace_handle_status: .event.name == "LIBP2P_TRACE_HANDLE_STATUS"
+ libp2p_trace_gossipsub_beacon_block: .event.name == "LIBP2P_TRACE_GOSSIPSUB_BEACON_BLOCK"
  xatu_server_events_router_matched:
  type: log_to_metric
  inputs:
@@ -406,6 +407,7 @@ transforms:
  - xatu_server_events_router.libp2p_trace_join
  - xatu_server_events_router.libp2p_trace_handle_metadata
  - xatu_server_events_router.libp2p_trace_handle_status
+ - xatu_server_events_router.libp2p_trace_gossipsub_beacon_block
  metrics:
  - type: counter
  field: event.name
@@ -2086,6 +2088,101 @@ transforms:
  del(.meta)
  del(.data)
  del(.path)
+
+ libp2p_trace_gossipsub_beacon_block_formatted:
+ type: remap
+ inputs:
+ - xatu_server_events_router.libp2p_trace_gossipsub_beacon_block
+ source: |-
+ event_date_time, err = parse_timestamp(.event.date_time, format: "%+");
+ if err == null {
+ .event_date_time = to_unix_timestamp(event_date_time, unit: "milliseconds")
+ } else {
+ .error = err
+ .error_description = "failed to parse event date time"
+ log(., level: "error", rate_limit_secs: 60)
+ }
+
+ peer_id_key, err = .meta.client.additional_data.metadata.peer_id + .meta.ethereum.network.name
+ if err != null {
+ .error = err
+ .error_description = "failed to generate peer id unique key"
+ log(., level: "error", rate_limit_secs: 60)
+ }
+ .peer_id_unique_key = seahash(peer_id_key)
+ .unique_key = seahash(.event.id)
+
+ .proposer_index = .data.proposer_index
+
+
+ .propagation_slot_start_diff = .meta.client.additional_data.propagation.slot_start_diff
+ .block = .data.block
+
+ .slot = .data.slot
+ slot_start_date_time, err = parse_timestamp(.meta.client.additional_data.slot.start_date_time, format: "%+");
+ if err == null {
+ .slot_start_date_time = to_unix_timestamp(slot_start_date_time)
+ } else {
+ .error = err
+ .error_description = "failed to parse slot start date time"
+ log(., level: "error", rate_limit_secs: 60)
+ }
+ .epoch = .meta.client.additional_data.epoch.number
+ epoch_start_date_time, err = parse_timestamp(.meta.client.additional_data.epoch.start_date_time, format: "%+");
+ if err == null {
+ .epoch_start_date_time = to_unix_timestamp(epoch_start_date_time)
+ } else {
+ .error = err
+ .error_description = "failed to parse epoch start date time"
+ log(., level: "error", rate_limit_secs: 60)
+ }
+
+ .wallclock_slot = .meta.client.additional_data.wallclock_slot.number
+ wallclock_slot_start_date_time, err = parse_timestamp(.meta.client.additional_data.wallclock_slot.start_date_time, format: "%+");
+ if err == null {
+ .wallclock_slot_start_date_time = to_unix_timestamp(wallclock_slot_start_date_time)
+ } else {
+ .error = err
+ .error_description = "failed to parse wallclock slot start date time"
+ log(., level: "error", rate_limit_secs: 60)
+ }
+ .wallclock_epoch = .meta.client.additional_data.epoch.number
+ wallclock_epoch_start_date_time, err = parse_timestamp(.meta.client.additional_data.wallclock_epoch.start_date_time, format: "%+");
+ if err == null {
+ .wallclock_epoch_start_date_time = to_unix_timestamp(wallclock_epoch_start_date_time)
+ } else {
+ .error = err
+ .error_description = "failed to parse wallclock epoch start date time"
+ log(., level: "error", rate_limit_secs: 60)
+ }
+
+ topicParts, err = split(.meta.client.additional_data.topic, "/")
+ if err != null {
+ .error = err
+ .error_description = "failed to split topic"
+ } else {
+ if length(topicParts) != 5 {
+ errDebug = {
+ "topic": .meta.client.additional_data.topic,
+ }
+ .error_description = "failed to split topic"
+ }
+ }
+
+ .topic_layer = topicParts[1]
+ .topic_fork_digest_value = topicParts[2]
+ .topic_name = topicParts[3]
+ .topic_encoding = topicParts[4]
+
+ .message_size = .meta.client.additional_data.message_size
+ .message_id = .meta.client.additional_data.message_id
+
+ .updated_date_time = to_unix_timestamp(now())
+
+ del(.event)
+ del(.meta)
+ del(.data)
+ del(.path)
  libp2p_trace_rpc_exploder:
  type: remap
  inputs:
@@ -3511,3 +3608,23 @@ sinks:
  healthcheck:
  enabled: true
  skip_unknown_fields: false
+ libp2p_trace_gossipsub_beacon_block_clickhouse:
+ type: clickhouse
+ inputs:
+ - libp2p_trace_gossipsub_beacon_block_formatted
+ database: default
+ endpoint: "${CLICKHOUSE_ENDPOINT}"
+ table: libp2p_gossipsub_beacon_block
+ auth:
+ strategy: basic
+ user: "${CLICKHOUSE_USER}"
+ password: "${CLICKHOUSE_PASSWORD}"
+ batch:
+ max_bytes: 52428800
+ max_events: 200000
+ timeout_secs: 1
+ buffer:
+ max_events: 200000
+ healthcheck:
+ enabled: true
+ skip_unknown_fields: false

deploy/migrations/clickhouse/034_libp2p_gossipsub_beacon_block.down.sql

@@ -0,0 +1,2 @@
+DROP TABLE IF EXISTS libp2p_gossipsub_beacon_block ON CLUSTER '{cluster}';
+DROP TABLE IF EXISTS libp2p_gossipsub_beacon_block_local ON CLUSTER '{cluster}';

deploy/migrations/clickhouse/034_libp2p_gossipsub_beacon_block.up.sql

@@ -0,0 +1,80 @@
+CREATE TABLE libp2p_gossipsub_beacon_block_local on cluster '{cluster}' (
+ event_date_time DateTime64(3) Codec(DoubleDelta, ZSTD(1)),
+ slot UInt32 Codec(DoubleDelta, ZSTD(1)),
+ slot_start_date_time DateTime Codec(DoubleDelta, ZSTD(1)),
+ epoch UInt32 Codec(DoubleDelta, ZSTD(1)),
+ epoch_start_date_time DateTime Codec(DoubleDelta, ZSTD(1)),
+ wallclock_slot UInt32 Codec(DoubleDelta, ZSTD(1)),
+ wallclock_slot_start_date_time DateTime Codec(DoubleDelta, ZSTD(1)),
+ wallclock_epoch UInt32 Codec(DoubleDelta, ZSTD(1)),
+ wallclock_epoch_start_date_time DateTime Codec(DoubleDelta, ZSTD(1)),
+ propagation_slot_start_diff UInt32 Codec(ZSTD(1)),
+ block FixedString(66) Codec(ZSTD(1)),
+ proposer_index UInt32 CODEC(ZSTD(1)),
+ peer_id_unique_key Int64,
+ message_id String CODEC(ZSTD(1)),
+ message_size UInt32 Codec(ZSTD(1)),
+ topic_layer LowCardinality(String),
+ topic_fork_digest_value LowCardinality(String),
+ topic_name LowCardinality(String),
+ topic_encoding LowCardinality(String),
+ meta_client_name LowCardinality(String),
+ meta_client_id String Codec(ZSTD(1)),
+ meta_client_version LowCardinality(String),
+ meta_client_implementation LowCardinality(String),
+ meta_client_os LowCardinality(String),
+ meta_client_ip Nullable(IPv6) Codec(ZSTD(1)),
+ meta_client_geo_city LowCardinality(String) Codec(ZSTD(1)),
+ meta_client_geo_country LowCardinality(String) Codec(ZSTD(1)),
+ meta_client_geo_country_code LowCardinality(String) Codec(ZSTD(1)),
+ meta_client_geo_continent_code LowCardinality(String) Codec(ZSTD(1)),
+ meta_client_geo_longitude Nullable(Float64) Codec(ZSTD(1)),
+ meta_client_geo_latitude Nullable(Float64) Codec(ZSTD(1)),
+ meta_client_geo_autonomous_system_number Nullable(UInt32) Codec(ZSTD(1)),
+ meta_client_geo_autonomous_system_organization Nullable(String) Codec(ZSTD(1)),
+ meta_network_id Int32 Codec(DoubleDelta, ZSTD(1)),
+ meta_network_name LowCardinality(String)
+) Engine = ReplicatedMergeTree('/clickhouse/{installation}/{cluster}/tables/{shard}/{database}/{table}', '{replica}')
+PARTITION BY toStartOfMonth(slot_start_date_time)
+ORDER BY (slot_start_date_time, meta_network_name, meta_client_name);
+
+ALTER TABLE libp2p_gossipsub_beacon_block_local ON CLUSTER '{cluster}'
+MODIFY COMMENT 'Table for libp2p gossipsub beacon block data.',
+COMMENT COLUMN event_date_time 'Timestamp of the event with millisecond precision',
+COMMENT COLUMN slot 'Slot number associated with the event',
+COMMENT COLUMN slot_start_date_time 'Start date and time of the slot',
+COMMENT COLUMN epoch 'Epoch number associated with the event',
+COMMENT COLUMN epoch_start_date_time 'Start date and time of the epoch',
+COMMENT COLUMN wallclock_slot 'Slot number of the wall clock when the event was received',
+COMMENT COLUMN wallclock_slot_start_date_time 'Start date and time of the wall clock slot when the event was received',
+COMMENT COLUMN wallclock_epoch 'Epoch number of the wall clock when the event was received',
+COMMENT COLUMN wallclock_epoch_start_date_time 'Start date and time of the wall clock epoch when the event was received',
+COMMENT COLUMN propagation_slot_start_diff 'Difference in slot start time for propagation',
+COMMENT COLUMN proposer_index 'The proposer index of the beacon block',
+COMMENT COLUMN block 'The beacon block root hash',
+COMMENT COLUMN peer_id_unique_key 'Unique key associated with the identifier of the peer',
+COMMENT COLUMN message_id 'Identifier of the message',
+COMMENT COLUMN message_size 'Size of the message in bytes',
+COMMENT COLUMN topic_layer 'Layer of the topic in the gossipsub protocol',
+COMMENT COLUMN topic_fork_digest_value 'Fork digest value of the topic',
+COMMENT COLUMN topic_name 'Name of the topic',
+COMMENT COLUMN topic_encoding 'Encoding used for the topic',
+COMMENT COLUMN meta_client_name 'Name of the client that generated the event',
+COMMENT COLUMN meta_client_id 'Unique Session ID of the client that generated the event. This changes every time the client is restarted.',
+COMMENT COLUMN meta_client_version 'Version of the client that generated the event',
+COMMENT COLUMN meta_client_implementation 'Implementation of the client that generated the event',
+COMMENT COLUMN meta_client_os 'Operating system of the client that generated the event',
+COMMENT COLUMN meta_client_ip 'IP address of the client that generated the event',
+COMMENT COLUMN meta_client_geo_city 'City of the client that generated the event',
+COMMENT COLUMN meta_client_geo_country 'Country of the client that generated the event',
+COMMENT COLUMN meta_client_geo_country_code 'Country code of the client that generated the event',
+COMMENT COLUMN meta_client_geo_continent_code 'Continent code of the client that generated the event',
+COMMENT COLUMN meta_client_geo_longitude 'Longitude of the client that generated the event',
+COMMENT COLUMN meta_client_geo_latitude 'Latitude of the client that generated the event',
+COMMENT COLUMN meta_client_geo_autonomous_system_number 'Autonomous system number of the client that generated the event',
+COMMENT COLUMN meta_client_geo_autonomous_system_organization 'Autonomous system organization of the client that generated the event',
+COMMENT COLUMN meta_network_id 'Network ID associated with the client',
+COMMENT COLUMN meta_network_name 'Name of the network associated with the client';
+
+CREATE TABLE libp2p_gossipsub_beacon_block on cluster '{cluster}' AS libp2p_gossipsub_beacon_block_local
+ENGINE = Distributed('{cluster}', default, libp2p_gossipsub_beacon_block_local, rand());

docker-compose.yml

@@ -248,6 +248,7 @@ services:
  "libp2p-trace-join"
  "libp2p-trace-handle-metadata"
  "libp2p-trace-handle-status"
+ "libp2p-trace-gossipsub-beacon-block"
  )
  for topic in "$${topics[@]}"; do
  echo "Creating topic: $$topic";

go.mod

@@ -2,7 +2,7 @@ module github.com/ethpandaops/xatu
 
 go 1.22.0
 
-replace github.com/probe-lab/hermes => github.com/ethpandaops/hermes v0.0.0-20240422024036-5bcb536550d4
+replace github.com/probe-lab/hermes => github.com/ethpandaops/hermes v0.0.0-20240424055612-d87881163f3b
 
 require (
  github.com/IBM/sarama v1.43.0

go.sum

@@ -262,14 +262,12 @@ github.com/ethereum/go-ethereum v1.13.14 h1:EwiY3FZP94derMCIam1iW4HFVrSgIcpsu0Hw
 github.com/ethereum/go-ethereum v1.13.14/go.mod h1:TN8ZiHrdJwSe8Cb6x+p0hs5CxhJZPbqB7hHkaUXcmIU=
 github.com/ethpandaops/beacon v0.35.0 h1:ZkHfxm41N0wkv503Xdb6rFxLuEnIonClUQWUPFHS5VU=
 github.com/ethpandaops/beacon v0.35.0/go.mod h1:B+SLxj1gnDd/Ia7cl/uuhzo1wyVf2p2puL6lmzPdPro=
-github.com/ethpandaops/ethcore v0.0.0-20240306031202-16f9e1926c0c h1:B/Auy5g6Nv4Pd8R9W9dC1IpysO7LzEmqb2C4iYM4j2A=
-github.com/ethpandaops/ethcore v0.0.0-20240306031202-16f9e1926c0c/go.mod h1:ZvKqL6CKxiraefdXPHeJurV2pDD/f2HF2uklDVdrry8=
 github.com/ethpandaops/ethcore v0.0.0-20240422023000-2a5727b18756 h1:8JWjrRfP14m0oxOk03m11n/xgdY5ceyUf/ZxYdOs5gE=
 github.com/ethpandaops/ethcore v0.0.0-20240422023000-2a5727b18756/go.mod h1:ZvKqL6CKxiraefdXPHeJurV2pDD/f2HF2uklDVdrry8=
 github.com/ethpandaops/ethwallclock v0.3.0 h1:xF5fwtBf+bHFHZKBnwiPFEuelW3sMM7SD3ZNFq1lJY4=
 github.com/ethpandaops/ethwallclock v0.3.0/go.mod h1:y0Cu+mhGLlem19vnAV2x0hpFS5KZ7oOi2SWYayv9l24=
-github.com/ethpandaops/hermes v0.0.0-20240422024036-5bcb536550d4 h1:y3MoajkTvlaNZFfEjGzI3U/EhBetpzDlntrEJjISPfU=
-github.com/ethpandaops/hermes v0.0.0-20240422024036-5bcb536550d4/go.mod h1:WOtvdP1zl53vuoOX6PRZ9oAbbJUiDso50efmAjoUZzI=
+github.com/ethpandaops/hermes v0.0.0-20240424055612-d87881163f3b h1:Iigwjof7KReOiEPHRAl3KWXFYDkVBN3GjxC8qwzdfwM=
+github.com/ethpandaops/hermes v0.0.0-20240424055612-d87881163f3b/go.mod h1:WOtvdP1zl53vuoOX6PRZ9oAbbJUiDso50efmAjoUZzI=
 github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
 github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=