From 3bb973c438c8e0df72c3963c3599008369eeee85 Mon Sep 17 00:00:00 2001 From: Andrew Davis <1709934+Savid@users.noreply.github.com> Date: Tue, 16 Jul 2024 16:23:37 +1000 Subject: [PATCH 1/3] feat(clickhouse): update canonical_beacon_validators table --- .../vector-kafka-clickhouse.yaml | 38 ++++++++--- ...anonical_beacon_validators_tweaks.down.sql | 56 ++++++++++++++++ ..._canonical_beacon_validators_tweaks.up.sql | 64 +++++++++++++++++++ 3 files changed, 149 insertions(+), 9 deletions(-) create mode 100644 deploy/migrations/clickhouse/043_canonical_beacon_validators_tweaks.down.sql create mode 100644 deploy/migrations/clickhouse/043_canonical_beacon_validators_tweaks.up.sql diff --git a/deploy/local/docker-compose/vector-kafka-clickhouse.yaml b/deploy/local/docker-compose/vector-kafka-clickhouse.yaml index 5cab8e5a..b0f486b6 100644 --- a/deploy/local/docker-compose/vector-kafka-clickhouse.yaml +++ b/deploy/local/docker-compose/vector-kafka-clickhouse.yaml @@ -1408,7 +1408,7 @@ transforms: .updated_date_time = to_unix_timestamp(now()) for_each(array!(.data.validators)) -> |_index, validator| { - events = push(events, { + event = { "event_date_time": .event_date_time, "updated_date_time": .updated_date_time, "meta_client_name": .meta_client_name, @@ -1430,15 +1430,35 @@ transforms: "epoch": .meta.client.additional_data.epoch.number, "epoch_start_date_time": .epoch_start_date_time, "index": validator.index, - "balance": validator.balance, "status": validator.status, - "activation_eligibility_epoch": validator.data.activation_eligibility_epoch, - "activation_epoch": validator.data.activation_epoch, - "effective_balance": validator.data.effective_balance, - "exit_epoch": validator.data.exit_epoch, - "slashed": validator.data.slashed, - "withdrawable_epoch": validator.data.withdrawable_epoch - }) + "slashed": validator.data.slashed + } + + if exists(validator.balance) && validator.balance != "0" { + event = merge(event, {"balance": validator.balance}) + } + + if exists(validator.effective_balance) && validator.effective_balance != "0" { + event = merge(event, {"effective_balance": validator.effective_balance}) + } + + if exists(validator.activation_epoch) && validator.activation_epoch != "0" { + event = merge(event, {"activation_epoch": validator.activation_epoch}) + } + + if exists(validator.activation_eligibility_epoch) && validator.activation_eligibility_epoch != "0" { + event = merge(event, {"activation_eligibility_epoch": validator.activation_eligibility_epoch}) + } + + if exists(validator.exit_epoch) && validator.exit_epoch != "0" && validator.exit_epoch != "18446744073709551615" { + event = merge(event, {"exit_epoch": validator.exit_epoch}) + } + + if exists(validator.withdrawable_epoch) && validator.withdrawable_epoch != "0" { + event = merge(event, {"withdrawable_epoch": validator.withdrawable_epoch}) + } + + events = push(events, event) } . = events canonical_beacon_validators_pubkeys_formatted: diff --git a/deploy/migrations/clickhouse/043_canonical_beacon_validators_tweaks.down.sql b/deploy/migrations/clickhouse/043_canonical_beacon_validators_tweaks.down.sql new file mode 100644 index 00000000..dbe0a8e4 --- /dev/null +++ b/deploy/migrations/clickhouse/043_canonical_beacon_validators_tweaks.down.sql @@ -0,0 +1,56 @@ +DROP TABLE IF EXISTS default.canonical_beacon_validators on cluster '{cluster}' SYNC; +DROP TABLE IF EXISTS default.canonical_beacon_validators_local on cluster '{cluster}' SYNC; + +CREATE TABLE default.canonical_beacon_validators_local +( + `updated_date_time` DateTime COMMENT 'When this row was last updated' CODEC(DoubleDelta, ZSTD(1)), + `event_date_time` DateTime64(3) COMMENT 'When the client fetched the beacon block from a beacon node' CODEC(DoubleDelta, ZSTD(1)), + `epoch` UInt32 COMMENT 'The epoch number from beacon block payload' CODEC(DoubleDelta, ZSTD(1)), + `epoch_start_date_time` DateTime COMMENT 'The wall clock time when the epoch started' CODEC(DoubleDelta, ZSTD(1)), + `index` UInt32 COMMENT 'The index of the validator' CODEC(ZSTD(1)), + `balance` UInt64 COMMENT 'The balance of the validator' CODEC(ZSTD(1)), + `status` LowCardinality(String) COMMENT 'The status of the validator', + `effective_balance` UInt64 COMMENT 'The effective balance of the validator' CODEC(ZSTD(1)), + `slashed` Bool COMMENT 'Whether the validator is slashed', + `activation_epoch` UInt64 COMMENT 'The epoch when the validator was activated' CODEC(ZSTD(1)), + `activation_eligibility_epoch` UInt64 COMMENT 'The epoch when the validator was activated' CODEC(ZSTD(1)), + `exit_epoch` UInt64 COMMENT 'The epoch when the validator exited' CODEC(ZSTD(1)), + `withdrawable_epoch` UInt64 COMMENT 'The epoch when the validator can withdraw' CODEC(ZSTD(1)), + `meta_client_name` LowCardinality(String) COMMENT 'Name of the client that generated the event', + `meta_client_id` String COMMENT 'Unique Session ID of the client that generated the event. This changes every time the client is restarted.' CODEC(ZSTD(1)), + `meta_client_version` LowCardinality(String) COMMENT 'Version of the client that generated the event', + `meta_client_implementation` LowCardinality(String) COMMENT 'Implementation of the client that generated the event', + `meta_client_os` LowCardinality(String) COMMENT 'Operating system of the client that generated the event', + `meta_client_ip` Nullable(IPv6) COMMENT 'IP address of the client that generated the event' CODEC(ZSTD(1)), + `meta_client_geo_city` LowCardinality(String) COMMENT 'City of the client that generated the event' CODEC(ZSTD(1)), + `meta_client_geo_country` LowCardinality(String) COMMENT 'Country of the client that generated the event' CODEC(ZSTD(1)), + `meta_client_geo_country_code` LowCardinality(String) COMMENT 'Country code of the client that generated the event' CODEC(ZSTD(1)), + `meta_client_geo_continent_code` LowCardinality(String) COMMENT 'Continent code of the client that generated the event' CODEC(ZSTD(1)), + `meta_client_geo_longitude` Nullable(Float64) COMMENT 'Longitude of the client that generated the event' CODEC(ZSTD(1)), + `meta_client_geo_latitude` Nullable(Float64) COMMENT 'Latitude of the client that generated the event' CODEC(ZSTD(1)), + `meta_client_geo_autonomous_system_number` Nullable(UInt32) COMMENT 'Autonomous system number of the client that generated the event' CODEC(ZSTD(1)), + `meta_client_geo_autonomous_system_organization` Nullable(String) COMMENT 'Autonomous system organization of the client that generated the event' CODEC(ZSTD(1)), + `meta_network_id` Int32 COMMENT 'Ethereum network ID' CODEC(DoubleDelta, ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name', + `meta_consensus_version` LowCardinality(String) COMMENT 'Ethereum consensus client version that generated the event', + `meta_consensus_version_major` LowCardinality(String) COMMENT 'Ethereum consensus client major version that generated the event', + `meta_consensus_version_minor` LowCardinality(String) COMMENT 'Ethereum consensus client minor version that generated the event', + `meta_consensus_version_patch` LowCardinality(String) COMMENT 'Ethereum consensus client patch version that generated the event', + `meta_consensus_implementation` LowCardinality(String) COMMENT 'Ethereum consensus client implementation that generated the event', + `meta_labels` Map(String, String) COMMENT 'Labels associated with the event' CODEC(ZSTD(1)) +) +ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{installation}/{cluster}/tables/{shard}/{database}/{table}', '{replica}', updated_date_time) +PARTITION BY toStartOfMonth(epoch_start_date_time) +ORDER BY (epoch_start_date_time, `index`, meta_network_name) +COMMENT 'Contains a validator state for an epoch.' + +CREATE TABLE default.canonical_beacon_validators ON CLUSTER '{cluster}' AS default.canonical_beacon_validators_local ENGINE = Distributed( + '{cluster}', + default, + canonical_beacon_validators_local, + cityHash64( + epoch_start_date_time, + `index`, + meta_network_name + ) +); diff --git a/deploy/migrations/clickhouse/043_canonical_beacon_validators_tweaks.up.sql b/deploy/migrations/clickhouse/043_canonical_beacon_validators_tweaks.up.sql new file mode 100644 index 00000000..3e0977f0 --- /dev/null +++ b/deploy/migrations/clickhouse/043_canonical_beacon_validators_tweaks.up.sql @@ -0,0 +1,64 @@ +DROP TABLE IF EXISTS default.canonical_beacon_validators on cluster '{cluster}' SYNC; + +DROP TABLE IF EXISTS default.canonical_beacon_validators_local on cluster '{cluster}' SYNC; + +CREATE TABLE default.canonical_beacon_validators_local ON CLUSTER '{cluster}' ( + `updated_date_time` DateTime COMMENT 'When this row was last updated' CODEC(DoubleDelta, ZSTD(1)), + `event_date_time` DateTime64(3) COMMENT 'When the client fetched the beacon block from a beacon node' CODEC(DoubleDelta, ZSTD(1)), + `epoch` UInt32 COMMENT 'The epoch number from beacon block payload' CODEC(DoubleDelta, ZSTD(1)), + `epoch_start_date_time` DateTime COMMENT 'The wall clock time when the epoch started' CODEC(DoubleDelta, ZSTD(1)), + `index` UInt32 COMMENT 'The index of the validator' CODEC(DoubleDelta, ZSTD(1)), + `balance` Nullable(UInt64) COMMENT 'The balance of the validator' CODEC(T64, ZSTD(1)), + `status` LowCardinality(String) COMMENT 'The status of the validator', + `effective_balance` Nullable(UInt64) COMMENT 'The effective balance of the validator' CODEC(ZSTD(1)), + `slashed` Bool COMMENT 'Whether the validator is slashed', + `activation_epoch` Nullable(UInt64) COMMENT 'The epoch when the validator was activated' CODEC(ZSTD(1)), + `activation_eligibility_epoch` Nullable(UInt64) COMMENT 'The epoch when the validator was activated' CODEC(ZSTD(1)), + `exit_epoch` Nullable(UInt64) COMMENT 'The epoch when the validator exited' CODEC(ZSTD(1)), + `withdrawable_epoch` Nullable(UInt64) COMMENT 'The epoch when the validator can withdraw' CODEC(ZSTD(1)), + `meta_client_name` LowCardinality(String) COMMENT 'Name of the client that generated the event', + `meta_client_id` String COMMENT 'Unique Session ID of the client that generated the event. This changes every time the client is restarted.' CODEC(ZSTD(1)), + `meta_client_version` LowCardinality(String) COMMENT 'Version of the client that generated the event', + `meta_client_implementation` LowCardinality(String) COMMENT 'Implementation of the client that generated the event', + `meta_client_os` LowCardinality(String) COMMENT 'Operating system of the client that generated the event', + `meta_client_ip` Nullable(IPv6) COMMENT 'IP address of the client that generated the event' CODEC(ZSTD(1)), + `meta_client_geo_city` LowCardinality(String) COMMENT 'City of the client that generated the event' CODEC(ZSTD(1)), + `meta_client_geo_country` LowCardinality(String) COMMENT 'Country of the client that generated the event' CODEC(ZSTD(1)), + `meta_client_geo_country_code` LowCardinality(String) COMMENT 'Country code of the client that generated the event' CODEC(ZSTD(1)), + `meta_client_geo_continent_code` LowCardinality(String) COMMENT 'Continent code of the client that generated the event' CODEC(ZSTD(1)), + `meta_client_geo_longitude` Nullable(Float64) COMMENT 'Longitude of the client that generated the event' CODEC(ZSTD(1)), + `meta_client_geo_latitude` Nullable(Float64) COMMENT 'Latitude of the client that generated the event' CODEC(ZSTD(1)), + `meta_client_geo_autonomous_system_number` Nullable(UInt32) COMMENT 'Autonomous system number of the client that generated the event' CODEC(ZSTD(1)), + `meta_client_geo_autonomous_system_organization` Nullable(String) COMMENT 'Autonomous system organization of the client that generated the event' CODEC(ZSTD(1)), + `meta_network_id` Int32 COMMENT 'Ethereum network ID' CODEC(DoubleDelta, ZSTD(1)), + `meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name', + `meta_consensus_version` LowCardinality(String) COMMENT 'Ethereum consensus client version that generated the event', + `meta_consensus_version_major` LowCardinality(String) COMMENT 'Ethereum consensus client major version that generated the event', + `meta_consensus_version_minor` LowCardinality(String) COMMENT 'Ethereum consensus client minor version that generated the event', + `meta_consensus_version_patch` LowCardinality(String) COMMENT 'Ethereum consensus client patch version that generated the event', + `meta_consensus_implementation` LowCardinality(String) COMMENT 'Ethereum consensus client implementation that generated the event', + `meta_labels` Map(String, String) COMMENT 'Labels associated with the event' CODEC(ZSTD(1)) +) ENGINE = ReplicatedReplacingMergeTree( + '/clickhouse/{installation}/{cluster}/{database}/tables/{table}/{shard}', + '{replica}', + updated_date_time +) PARTITION BY toStartOfMonth(epoch_start_date_time) +ORDER BY + ( + epoch_start_date_time, + meta_network_name, + `index`, + `status` + ) COMMENT 'Contains a validator state for an epoch.'; + +CREATE TABLE default.canonical_beacon_validators ON CLUSTER '{cluster}' AS default.canonical_beacon_validators_local ENGINE = Distributed( + '{cluster}', + default, + canonical_beacon_validators_local, + cityHash64( + epoch_start_date_time, + meta_network_name, + `index`, + `status` + ) +); From a436c5a5f0b9a2cd448f4dd6b9420140f005e70d Mon Sep 17 00:00:00 2001 From: Andrew Davis <1709934+Savid@users.noreply.github.com> Date: Wed, 17 Jul 2024 09:54:41 +1000 Subject: [PATCH 2/3] fix vector --- .../vector-kafka-clickhouse.yaml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/deploy/local/docker-compose/vector-kafka-clickhouse.yaml b/deploy/local/docker-compose/vector-kafka-clickhouse.yaml index b0f486b6..cd0b9dce 100644 --- a/deploy/local/docker-compose/vector-kafka-clickhouse.yaml +++ b/deploy/local/docker-compose/vector-kafka-clickhouse.yaml @@ -1438,24 +1438,24 @@ transforms: event = merge(event, {"balance": validator.balance}) } - if exists(validator.effective_balance) && validator.effective_balance != "0" { - event = merge(event, {"effective_balance": validator.effective_balance}) + if exists(validator.data.effective_balance) && validator.data.effective_balance != "0" { + event = merge(event, {"effective_balance": validator.data.effective_balance}) } - if exists(validator.activation_epoch) && validator.activation_epoch != "0" { - event = merge(event, {"activation_epoch": validator.activation_epoch}) + if exists(validator.data.activation_epoch) && validator.data.activation_epoch != "0" && validator.data.activation_epoch != "18446744073709551615" { + event = merge(event, {"activation_epoch": validator.data.activation_epoch}) } - if exists(validator.activation_eligibility_epoch) && validator.activation_eligibility_epoch != "0" { - event = merge(event, {"activation_eligibility_epoch": validator.activation_eligibility_epoch}) + if exists(validator.data.activation_eligibility_epoch) && validator.data.activation_eligibility_epoch != "0" && validator.data.activation_eligibility_epoch != "18446744073709551615" { + event = merge(event, {"activation_eligibility_epoch": validator.data.activation_eligibility_epoch}) } - if exists(validator.exit_epoch) && validator.exit_epoch != "0" && validator.exit_epoch != "18446744073709551615" { - event = merge(event, {"exit_epoch": validator.exit_epoch}) + if exists(validator.data.exit_epoch) && validator.data.exit_epoch != "0" && validator.data.exit_epoch != "18446744073709551615" { + event = merge(event, {"exit_epoch": validator.data.exit_epoch}) } - if exists(validator.withdrawable_epoch) && validator.withdrawable_epoch != "0" { - event = merge(event, {"withdrawable_epoch": validator.withdrawable_epoch}) + if exists(validator.data.withdrawable_epoch) && validator.data.withdrawable_epoch != "0" && validator.data.withdrawable_epoch != "18446744073709551615" { + event = merge(event, {"withdrawable_epoch": validator.data.withdrawable_epoch}) } events = push(events, event) From 052dcd0f9e20ed44ff614c963d084b4b00763236 Mon Sep 17 00:00:00 2001 From: Andrew Davis <1709934+Savid@users.noreply.github.com> Date: Wed, 17 Jul 2024 12:15:15 +1000 Subject: [PATCH 3/3] fix vector --- .../vector-kafka-clickhouse.yaml | 18 ++++++++++++++++++ ...3_canonical_beacon_validators_tweaks.up.sql | 4 ++++ 2 files changed, 22 insertions(+) diff --git a/deploy/local/docker-compose/vector-kafka-clickhouse.yaml b/deploy/local/docker-compose/vector-kafka-clickhouse.yaml index cd0b9dce..773057a6 100644 --- a/deploy/local/docker-compose/vector-kafka-clickhouse.yaml +++ b/deploy/local/docker-compose/vector-kafka-clickhouse.yaml @@ -1427,6 +1427,12 @@ transforms: "meta_client_geo_latitude": .meta_client_geo_latitude, "meta_client_geo_autonomous_system_number": .meta_client_geo_autonomous_system_number, "meta_client_geo_autonomous_system_organization": .meta_client_geo_autonomous_system_organization, + "meta_consensus_version": .meta_consensus_version, + "meta_consensus_version_major": .meta_consensus_version_major, + "meta_consensus_version_minor": .meta_consensus_version_minor, + "meta_consensus_version_patch": .meta_consensus_version_patch, + "meta_consensus_implementation": .meta_consensus_implementation, + "meta_labels": .meta_labels, "epoch": .meta.client.additional_data.epoch.number, "epoch_start_date_time": .epoch_start_date_time, "index": validator.index, @@ -1508,6 +1514,12 @@ transforms: "meta_client_geo_latitude": .meta_client_geo_latitude, "meta_client_geo_autonomous_system_number": .meta_client_geo_autonomous_system_number, "meta_client_geo_autonomous_system_organization": .meta_client_geo_autonomous_system_organization, + "meta_consensus_version": .meta_consensus_version, + "meta_consensus_version_major": .meta_consensus_version_major, + "meta_consensus_version_minor": .meta_consensus_version_minor, + "meta_consensus_version_patch": .meta_consensus_version_patch, + "meta_consensus_implementation": .meta_consensus_implementation, + "meta_labels": .meta_labels, "epoch": .meta.client.additional_data.epoch.number, "epoch_start_date_time": .epoch_start_date_time, "index": validator.index, @@ -1562,6 +1574,12 @@ transforms: "meta_client_geo_latitude": .meta_client_geo_latitude, "meta_client_geo_autonomous_system_number": .meta_client_geo_autonomous_system_number, "meta_client_geo_autonomous_system_organization": .meta_client_geo_autonomous_system_organization, + "meta_consensus_version": .meta_consensus_version, + "meta_consensus_version_major": .meta_consensus_version_major, + "meta_consensus_version_minor": .meta_consensus_version_minor, + "meta_consensus_version_patch": .meta_consensus_version_patch, + "meta_consensus_implementation": .meta_consensus_implementation, + "meta_labels": .meta_labels, "epoch": .meta.client.additional_data.epoch.number, "epoch_start_date_time": .epoch_start_date_time, "index": validator.index, diff --git a/deploy/migrations/clickhouse/043_canonical_beacon_validators_tweaks.up.sql b/deploy/migrations/clickhouse/043_canonical_beacon_validators_tweaks.up.sql index 3e0977f0..fd47e771 100644 --- a/deploy/migrations/clickhouse/043_canonical_beacon_validators_tweaks.up.sql +++ b/deploy/migrations/clickhouse/043_canonical_beacon_validators_tweaks.up.sql @@ -2,6 +2,10 @@ DROP TABLE IF EXISTS default.canonical_beacon_validators on cluster '{cluster}' DROP TABLE IF EXISTS default.canonical_beacon_validators_local on cluster '{cluster}' SYNC; +TRUNCATE TABLE canonical_beacon_validators_pubkeys_local ON CLUSTER '{cluster}'; + +TRUNCATE TABLE canonical_beacon_validators_withdrawal_credentials_local ON CLUSTER '{cluster}'; + CREATE TABLE default.canonical_beacon_validators_local ON CLUSTER '{cluster}' ( `updated_date_time` DateTime COMMENT 'When this row was last updated' CODEC(DoubleDelta, ZSTD(1)), `event_date_time` DateTime64(3) COMMENT 'When the client fetched the beacon block from a beacon node' CODEC(DoubleDelta, ZSTD(1)),