Skip to content

Commit

Permalink
Merge pull request #348 from ethpandaops/feat/canonical-beacon-valida…
Browse files Browse the repository at this point in the history 
…tors

feat(clickhouse): update canonical_beacon_validators table
  • Loading branch information
@Savid
Savid committed Jul 17, 2024
2 parents abc3835 + 052dcd0 commit c259c50
Show file tree
Hide file tree
Showing 3 changed files with 171 additions and 9 deletions.
56 changes: 47 additions & 9 deletions deploy/local/docker-compose/vector-kafka-clickhouse.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1408,7 +1408,7 @@ transforms:
.updated_date_time = to_unix_timestamp(now())
for_each(array!(.data.validators)) -> |_index, validator| {
events = push(events, {
event = {
"event_date_time": .event_date_time,
"updated_date_time": .updated_date_time,
"meta_client_name": .meta_client_name,
Expand All @@ -1427,18 +1427,44 @@ transforms:
"meta_client_geo_latitude": .meta_client_geo_latitude,
"meta_client_geo_autonomous_system_number": .meta_client_geo_autonomous_system_number,
"meta_client_geo_autonomous_system_organization": .meta_client_geo_autonomous_system_organization,
"meta_consensus_version": .meta_consensus_version,
"meta_consensus_version_major": .meta_consensus_version_major,
"meta_consensus_version_minor": .meta_consensus_version_minor,
"meta_consensus_version_patch": .meta_consensus_version_patch,
"meta_consensus_implementation": .meta_consensus_implementation,
"meta_labels": .meta_labels,
"epoch": .meta.client.additional_data.epoch.number,
"epoch_start_date_time": .epoch_start_date_time,
"index": validator.index,
"balance": validator.balance,
"status": validator.status,
"activation_eligibility_epoch": validator.data.activation_eligibility_epoch,
"activation_epoch": validator.data.activation_epoch,
"effective_balance": validator.data.effective_balance,
"exit_epoch": validator.data.exit_epoch,
"slashed": validator.data.slashed,
"withdrawable_epoch": validator.data.withdrawable_epoch
})
"slashed": validator.data.slashed
}
if exists(validator.balance) && validator.balance != "0" {
event = merge(event, {"balance": validator.balance})
}
if exists(validator.data.effective_balance) && validator.data.effective_balance != "0" {
event = merge(event, {"effective_balance": validator.data.effective_balance})
}
if exists(validator.data.activation_epoch) && validator.data.activation_epoch != "0" && validator.data.activation_epoch != "18446744073709551615" {
event = merge(event, {"activation_epoch": validator.data.activation_epoch})
}
if exists(validator.data.activation_eligibility_epoch) && validator.data.activation_eligibility_epoch != "0" && validator.data.activation_eligibility_epoch != "18446744073709551615" {
event = merge(event, {"activation_eligibility_epoch": validator.data.activation_eligibility_epoch})
}
if exists(validator.data.exit_epoch) && validator.data.exit_epoch != "0" && validator.data.exit_epoch != "18446744073709551615" {
event = merge(event, {"exit_epoch": validator.data.exit_epoch})
}
if exists(validator.data.withdrawable_epoch) && validator.data.withdrawable_epoch != "0" && validator.data.withdrawable_epoch != "18446744073709551615" {
event = merge(event, {"withdrawable_epoch": validator.data.withdrawable_epoch})
}
events = push(events, event)
}
. = events
canonical_beacon_validators_pubkeys_formatted:
Expand Down Expand Up @@ -1488,6 +1514,12 @@ transforms:
"meta_client_geo_latitude": .meta_client_geo_latitude,
"meta_client_geo_autonomous_system_number": .meta_client_geo_autonomous_system_number,
"meta_client_geo_autonomous_system_organization": .meta_client_geo_autonomous_system_organization,
"meta_consensus_version": .meta_consensus_version,
"meta_consensus_version_major": .meta_consensus_version_major,
"meta_consensus_version_minor": .meta_consensus_version_minor,
"meta_consensus_version_patch": .meta_consensus_version_patch,
"meta_consensus_implementation": .meta_consensus_implementation,
"meta_labels": .meta_labels,
"epoch": .meta.client.additional_data.epoch.number,
"epoch_start_date_time": .epoch_start_date_time,
"index": validator.index,
Expand Down Expand Up @@ -1542,6 +1574,12 @@ transforms:
"meta_client_geo_latitude": .meta_client_geo_latitude,
"meta_client_geo_autonomous_system_number": .meta_client_geo_autonomous_system_number,
"meta_client_geo_autonomous_system_organization": .meta_client_geo_autonomous_system_organization,
"meta_consensus_version": .meta_consensus_version,
"meta_consensus_version_major": .meta_consensus_version_major,
"meta_consensus_version_minor": .meta_consensus_version_minor,
"meta_consensus_version_patch": .meta_consensus_version_patch,
"meta_consensus_implementation": .meta_consensus_implementation,
"meta_labels": .meta_labels,
"epoch": .meta.client.additional_data.epoch.number,
"epoch_start_date_time": .epoch_start_date_time,
"index": validator.index,
Expand Down
56 changes: 56 additions & 0 deletions deploy/migrations/clickhouse/043_canonical_beacon_validators_tweaks.down.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
DROP TABLE IF EXISTS default.canonical_beacon_validators on cluster '{cluster}' SYNC;
DROP TABLE IF EXISTS default.canonical_beacon_validators_local on cluster '{cluster}' SYNC;

CREATE TABLE default.canonical_beacon_validators_local
(
`updated_date_time` DateTime COMMENT 'When this row was last updated' CODEC(DoubleDelta, ZSTD(1)),
`event_date_time` DateTime64(3) COMMENT 'When the client fetched the beacon block from a beacon node' CODEC(DoubleDelta, ZSTD(1)),
`epoch` UInt32 COMMENT 'The epoch number from beacon block payload' CODEC(DoubleDelta, ZSTD(1)),
`epoch_start_date_time` DateTime COMMENT 'The wall clock time when the epoch started' CODEC(DoubleDelta, ZSTD(1)),
`index` UInt32 COMMENT 'The index of the validator' CODEC(ZSTD(1)),
`balance` UInt64 COMMENT 'The balance of the validator' CODEC(ZSTD(1)),
`status` LowCardinality(String) COMMENT 'The status of the validator',
`effective_balance` UInt64 COMMENT 'The effective balance of the validator' CODEC(ZSTD(1)),
`slashed` Bool COMMENT 'Whether the validator is slashed',
`activation_epoch` UInt64 COMMENT 'The epoch when the validator was activated' CODEC(ZSTD(1)),
`activation_eligibility_epoch` UInt64 COMMENT 'The epoch when the validator was activated' CODEC(ZSTD(1)),
`exit_epoch` UInt64 COMMENT 'The epoch when the validator exited' CODEC(ZSTD(1)),
`withdrawable_epoch` UInt64 COMMENT 'The epoch when the validator can withdraw' CODEC(ZSTD(1)),
`meta_client_name` LowCardinality(String) COMMENT 'Name of the client that generated the event',
`meta_client_id` String COMMENT 'Unique Session ID of the client that generated the event. This changes every time the client is restarted.' CODEC(ZSTD(1)),
`meta_client_version` LowCardinality(String) COMMENT 'Version of the client that generated the event',
`meta_client_implementation` LowCardinality(String) COMMENT 'Implementation of the client that generated the event',
`meta_client_os` LowCardinality(String) COMMENT 'Operating system of the client that generated the event',
`meta_client_ip` Nullable(IPv6) COMMENT 'IP address of the client that generated the event' CODEC(ZSTD(1)),
`meta_client_geo_city` LowCardinality(String) COMMENT 'City of the client that generated the event' CODEC(ZSTD(1)),
`meta_client_geo_country` LowCardinality(String) COMMENT 'Country of the client that generated the event' CODEC(ZSTD(1)),
`meta_client_geo_country_code` LowCardinality(String) COMMENT 'Country code of the client that generated the event' CODEC(ZSTD(1)),
`meta_client_geo_continent_code` LowCardinality(String) COMMENT 'Continent code of the client that generated the event' CODEC(ZSTD(1)),
`meta_client_geo_longitude` Nullable(Float64) COMMENT 'Longitude of the client that generated the event' CODEC(ZSTD(1)),
`meta_client_geo_latitude` Nullable(Float64) COMMENT 'Latitude of the client that generated the event' CODEC(ZSTD(1)),
`meta_client_geo_autonomous_system_number` Nullable(UInt32) COMMENT 'Autonomous system number of the client that generated the event' CODEC(ZSTD(1)),
`meta_client_geo_autonomous_system_organization` Nullable(String) COMMENT 'Autonomous system organization of the client that generated the event' CODEC(ZSTD(1)),
`meta_network_id` Int32 COMMENT 'Ethereum network ID' CODEC(DoubleDelta, ZSTD(1)),
`meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name',
`meta_consensus_version` LowCardinality(String) COMMENT 'Ethereum consensus client version that generated the event',
`meta_consensus_version_major` LowCardinality(String) COMMENT 'Ethereum consensus client major version that generated the event',
`meta_consensus_version_minor` LowCardinality(String) COMMENT 'Ethereum consensus client minor version that generated the event',
`meta_consensus_version_patch` LowCardinality(String) COMMENT 'Ethereum consensus client patch version that generated the event',
`meta_consensus_implementation` LowCardinality(String) COMMENT 'Ethereum consensus client implementation that generated the event',
`meta_labels` Map(String, String) COMMENT 'Labels associated with the event' CODEC(ZSTD(1))
)
ENGINE = ReplicatedReplacingMergeTree('/clickhouse/{installation}/{cluster}/tables/{shard}/{database}/{table}', '{replica}', updated_date_time)
PARTITION BY toStartOfMonth(epoch_start_date_time)
ORDER BY (epoch_start_date_time, `index`, meta_network_name)
COMMENT 'Contains a validator state for an epoch.'

CREATE TABLE default.canonical_beacon_validators ON CLUSTER '{cluster}' AS default.canonical_beacon_validators_local ENGINE = Distributed(
'{cluster}',
default,
canonical_beacon_validators_local,
cityHash64(
epoch_start_date_time,
`index`,
meta_network_name
)
);
68 changes: 68 additions & 0 deletions deploy/migrations/clickhouse/043_canonical_beacon_validators_tweaks.up.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
DROP TABLE IF EXISTS default.canonical_beacon_validators on cluster '{cluster}' SYNC;

DROP TABLE IF EXISTS default.canonical_beacon_validators_local on cluster '{cluster}' SYNC;

TRUNCATE TABLE canonical_beacon_validators_pubkeys_local ON CLUSTER '{cluster}';

TRUNCATE TABLE canonical_beacon_validators_withdrawal_credentials_local ON CLUSTER '{cluster}';

CREATE TABLE default.canonical_beacon_validators_local ON CLUSTER '{cluster}' (
`updated_date_time` DateTime COMMENT 'When this row was last updated' CODEC(DoubleDelta, ZSTD(1)),
`event_date_time` DateTime64(3) COMMENT 'When the client fetched the beacon block from a beacon node' CODEC(DoubleDelta, ZSTD(1)),
`epoch` UInt32 COMMENT 'The epoch number from beacon block payload' CODEC(DoubleDelta, ZSTD(1)),
`epoch_start_date_time` DateTime COMMENT 'The wall clock time when the epoch started' CODEC(DoubleDelta, ZSTD(1)),
`index` UInt32 COMMENT 'The index of the validator' CODEC(DoubleDelta, ZSTD(1)),
`balance` Nullable(UInt64) COMMENT 'The balance of the validator' CODEC(T64, ZSTD(1)),
`status` LowCardinality(String) COMMENT 'The status of the validator',
`effective_balance` Nullable(UInt64) COMMENT 'The effective balance of the validator' CODEC(ZSTD(1)),
`slashed` Bool COMMENT 'Whether the validator is slashed',
`activation_epoch` Nullable(UInt64) COMMENT 'The epoch when the validator was activated' CODEC(ZSTD(1)),
`activation_eligibility_epoch` Nullable(UInt64) COMMENT 'The epoch when the validator was activated' CODEC(ZSTD(1)),
`exit_epoch` Nullable(UInt64) COMMENT 'The epoch when the validator exited' CODEC(ZSTD(1)),
`withdrawable_epoch` Nullable(UInt64) COMMENT 'The epoch when the validator can withdraw' CODEC(ZSTD(1)),
`meta_client_name` LowCardinality(String) COMMENT 'Name of the client that generated the event',
`meta_client_id` String COMMENT 'Unique Session ID of the client that generated the event. This changes every time the client is restarted.' CODEC(ZSTD(1)),
`meta_client_version` LowCardinality(String) COMMENT 'Version of the client that generated the event',
`meta_client_implementation` LowCardinality(String) COMMENT 'Implementation of the client that generated the event',
`meta_client_os` LowCardinality(String) COMMENT 'Operating system of the client that generated the event',
`meta_client_ip` Nullable(IPv6) COMMENT 'IP address of the client that generated the event' CODEC(ZSTD(1)),
`meta_client_geo_city` LowCardinality(String) COMMENT 'City of the client that generated the event' CODEC(ZSTD(1)),
`meta_client_geo_country` LowCardinality(String) COMMENT 'Country of the client that generated the event' CODEC(ZSTD(1)),
`meta_client_geo_country_code` LowCardinality(String) COMMENT 'Country code of the client that generated the event' CODEC(ZSTD(1)),
`meta_client_geo_continent_code` LowCardinality(String) COMMENT 'Continent code of the client that generated the event' CODEC(ZSTD(1)),
`meta_client_geo_longitude` Nullable(Float64) COMMENT 'Longitude of the client that generated the event' CODEC(ZSTD(1)),
`meta_client_geo_latitude` Nullable(Float64) COMMENT 'Latitude of the client that generated the event' CODEC(ZSTD(1)),
`meta_client_geo_autonomous_system_number` Nullable(UInt32) COMMENT 'Autonomous system number of the client that generated the event' CODEC(ZSTD(1)),
`meta_client_geo_autonomous_system_organization` Nullable(String) COMMENT 'Autonomous system organization of the client that generated the event' CODEC(ZSTD(1)),
`meta_network_id` Int32 COMMENT 'Ethereum network ID' CODEC(DoubleDelta, ZSTD(1)),
`meta_network_name` LowCardinality(String) COMMENT 'Ethereum network name',
`meta_consensus_version` LowCardinality(String) COMMENT 'Ethereum consensus client version that generated the event',
`meta_consensus_version_major` LowCardinality(String) COMMENT 'Ethereum consensus client major version that generated the event',
`meta_consensus_version_minor` LowCardinality(String) COMMENT 'Ethereum consensus client minor version that generated the event',
`meta_consensus_version_patch` LowCardinality(String) COMMENT 'Ethereum consensus client patch version that generated the event',
`meta_consensus_implementation` LowCardinality(String) COMMENT 'Ethereum consensus client implementation that generated the event',
`meta_labels` Map(String, String) COMMENT 'Labels associated with the event' CODEC(ZSTD(1))
) ENGINE = ReplicatedReplacingMergeTree(
'/clickhouse/{installation}/{cluster}/{database}/tables/{table}/{shard}',
'{replica}',
updated_date_time
) PARTITION BY toStartOfMonth(epoch_start_date_time)
ORDER BY
(
epoch_start_date_time,
meta_network_name,
`index`,
`status`
) COMMENT 'Contains a validator state for an epoch.';

CREATE TABLE default.canonical_beacon_validators ON CLUSTER '{cluster}' AS default.canonical_beacon_validators_local ENGINE = Distributed(
'{cluster}',
default,
canonical_beacon_validators_local,
cityHash64(
epoch_start_date_time,
meta_network_name,
`index`,
`status`
)
);

0 comments on commit c259c50

Please sign in to comment.