From e52ced1d3ea3664d32fc5770c99809694b27c29a Mon Sep 17 00:00:00 2001 From: Andrew Davis <1709934+Savid@users.noreply.github.com> Date: Tue, 26 Mar 2024 09:49:35 +1000 Subject: [PATCH] feat(deploy/migrations): add comments --- deploy/migrations/clickhouse/004_dbt.up.sql | 2 +- deploy/migrations/clickhouse/007_tmp.up.sql | 2 +- .../migrations/clickhouse/020_imports.up.sql | 2 +- .../clickhouse/028_comments.down.sql | 75 +++++++++++++++++++ .../migrations/clickhouse/028_comments.up.sql | 75 +++++++++++++++++++ ...x_replacing_tables_sharding_keys.down.sql} | 0 ...fix_replacing_tables_sharding_keys.up.sql} | 0 7 files changed, 153 insertions(+), 3 deletions(-) create mode 100644 deploy/migrations/clickhouse/028_comments.down.sql create mode 100644 deploy/migrations/clickhouse/028_comments.up.sql rename deploy/migrations/clickhouse/{028_fix_replacing_tables_sharding_keys.down.sql => 029_fix_replacing_tables_sharding_keys.down.sql} (100%) rename deploy/migrations/clickhouse/{028_fix_replacing_tables_sharding_keys.up.sql => 029_fix_replacing_tables_sharding_keys.up.sql} (100%) diff --git a/deploy/migrations/clickhouse/004_dbt.up.sql b/deploy/migrations/clickhouse/004_dbt.up.sql index 55e067f1..e229b8c3 100644 --- a/deploy/migrations/clickhouse/004_dbt.up.sql +++ b/deploy/migrations/clickhouse/004_dbt.up.sql @@ -1 +1 @@ -CREATE DATABASE dbt ON CLUSTER '{cluster}' +CREATE DATABASE IF NOT EXISTS dbt ON CLUSTER '{cluster}' diff --git a/deploy/migrations/clickhouse/007_tmp.up.sql b/deploy/migrations/clickhouse/007_tmp.up.sql index 29fb7bae..757da6cd 100644 --- a/deploy/migrations/clickhouse/007_tmp.up.sql +++ b/deploy/migrations/clickhouse/007_tmp.up.sql @@ -1 +1 @@ -CREATE DATABASE tmp ON CLUSTER '{cluster}' +CREATE DATABASE IF NOT EXISTS tmp ON CLUSTER '{cluster}' diff --git a/deploy/migrations/clickhouse/020_imports.up.sql b/deploy/migrations/clickhouse/020_imports.up.sql index 6727a99e..7066d03e 100644 --- a/deploy/migrations/clickhouse/020_imports.up.sql +++ b/deploy/migrations/clickhouse/020_imports.up.sql @@ -42,7 +42,7 @@ PARTITION BY toStartOfMonth(timestamp) ORDER BY (timestamp, unique_key, chain_id); ALTER TABLE default.mempool_dumpster_transaction_local ON CLUSTER '{cluster}' -MODIFY COMMENT 'Contains transactions from mempool dumpster dataset. Following the parquet schema with some additions: https://github.com/flashbots/mempool-dumpster?tab=readme-ov-file#schema-of-output-files', +MODIFY COMMENT 'Contains transactions from mempool dumpster dataset. Following the parquet schema with some additions', COMMENT COLUMN unique_key 'Unique key for the row, this is outside the source data and used for deduplication', COMMENT COLUMN updated_date_time 'When this row was last updated, this is outside the source data and used for deduplication', COMMENT COLUMN timestamp 'Timestamp of the transaction', diff --git a/deploy/migrations/clickhouse/028_comments.down.sql b/deploy/migrations/clickhouse/028_comments.down.sql new file mode 100644 index 00000000..b89fa09c --- /dev/null +++ b/deploy/migrations/clickhouse/028_comments.down.sql @@ -0,0 +1,75 @@ +ALTER TABLE default.beacon_api_eth_v1_beacon_committee ON CLUSTER '{cluster}' +MODIFY COMMENT '', +COMMENT COLUMN event_date_time '', +COMMENT COLUMN slot '', +COMMENT COLUMN slot_start_date_time '', +COMMENT COLUMN committee_index '', +COMMENT COLUMN validators '', +COMMENT COLUMN epoch '', +COMMENT COLUMN epoch_start_date_time '', +COMMENT COLUMN meta_client_name '', +COMMENT COLUMN meta_client_id '', +COMMENT COLUMN meta_client_version '', +COMMENT COLUMN meta_client_implementation '', +COMMENT COLUMN meta_client_os '', +COMMENT COLUMN meta_client_ip '', +COMMENT COLUMN meta_client_geo_city '', +COMMENT COLUMN meta_client_geo_country '', +COMMENT COLUMN meta_client_geo_country_code '', +COMMENT COLUMN meta_client_geo_continent_code '', +COMMENT COLUMN meta_client_geo_longitude '', +COMMENT COLUMN meta_client_geo_latitude '', +COMMENT COLUMN meta_client_geo_autonomous_system_number '', +COMMENT COLUMN meta_client_geo_autonomous_system_organization '', +COMMENT COLUMN meta_network_id '', +COMMENT COLUMN meta_network_name '', +COMMENT COLUMN meta_consensus_version '', +COMMENT COLUMN meta_consensus_version_major '', +COMMENT COLUMN meta_consensus_version_minor '', +COMMENT COLUMN meta_consensus_version_patch '', +COMMENT COLUMN meta_consensus_implementation '', +COMMENT COLUMN meta_labels ''; + +ALTER TABLE default.beacon_api_eth_v1_beacon_committee_local ON CLUSTER '{cluster}' +MODIFY COMMENT '', +COMMENT COLUMN event_date_time '', +COMMENT COLUMN slot '', +COMMENT COLUMN slot_start_date_time '', +COMMENT COLUMN committee_index '', +COMMENT COLUMN validators '', +COMMENT COLUMN epoch '', +COMMENT COLUMN epoch_start_date_time '', +COMMENT COLUMN meta_client_name '', +COMMENT COLUMN meta_client_id '', +COMMENT COLUMN meta_client_version '', +COMMENT COLUMN meta_client_implementation '', +COMMENT COLUMN meta_client_os '', +COMMENT COLUMN meta_client_ip '', +COMMENT COLUMN meta_client_geo_city '', +COMMENT COLUMN meta_client_geo_country '', +COMMENT COLUMN meta_client_geo_country_code '', +COMMENT COLUMN meta_client_geo_continent_code '', +COMMENT COLUMN meta_client_geo_longitude '', +COMMENT COLUMN meta_client_geo_latitude '', +COMMENT COLUMN meta_client_geo_autonomous_system_number '', +COMMENT COLUMN meta_client_geo_autonomous_system_organization '', +COMMENT COLUMN meta_network_id '', +COMMENT COLUMN meta_network_name '', +COMMENT COLUMN meta_consensus_version '', +COMMENT COLUMN meta_consensus_version_major '', +COMMENT COLUMN meta_consensus_version_minor '', +COMMENT COLUMN meta_consensus_version_patch '', +COMMENT COLUMN meta_consensus_implementation '', +COMMENT COLUMN meta_labels ''; + +ALTER TABLE default.beacon_api_eth_v1_events_blob_sidecar ON CLUSTER '{cluster}' +MODIFY COMMENT ''; + +ALTER TABLE default.beacon_api_eth_v1_events_blob_sidecar_local ON CLUSTER '{cluster}' +MODIFY COMMENT ''; + +ALTER TABLE default.beacon_api_eth_v1_validator_attestation_data ON CLUSTER '{cluster}' +MODIFY COMMENT ''; + +ALTER TABLE default.beacon_api_eth_v1_validator_attestation_data_local ON CLUSTER '{cluster}' +MODIFY COMMENT ''; diff --git a/deploy/migrations/clickhouse/028_comments.up.sql b/deploy/migrations/clickhouse/028_comments.up.sql new file mode 100644 index 00000000..a2565af0 --- /dev/null +++ b/deploy/migrations/clickhouse/028_comments.up.sql @@ -0,0 +1,75 @@ +ALTER TABLE default.beacon_api_eth_v1_beacon_committee ON CLUSTER '{cluster}' +MODIFY COMMENT 'Contains beacon API /eth/v1/beacon/states/{state_id}/committees data from each sentry client attached to a beacon node.', +COMMENT COLUMN event_date_time 'When the sentry received the event from a beacon node', +COMMENT COLUMN slot 'Slot number in the beacon API committee payload', +COMMENT COLUMN slot_start_date_time 'The wall clock time when the slot started', +COMMENT COLUMN committee_index 'The committee index in the beacon API committee payload', +COMMENT COLUMN validators 'The validator indices in the beacon API committee payload', +COMMENT COLUMN epoch 'The epoch number in the beacon API committee payload', +COMMENT COLUMN epoch_start_date_time 'The wall clock time when the epoch started', +COMMENT COLUMN meta_client_name 'Name of the client that generated the event', +COMMENT COLUMN meta_client_id 'Unique Session ID of the client that generated the event. This changes every time the client is restarted.', +COMMENT COLUMN meta_client_version 'Version of the client that generated the event', +COMMENT COLUMN meta_client_implementation 'Implementation of the client that generated the event', +COMMENT COLUMN meta_client_os 'Operating system of the client that generated the event', +COMMENT COLUMN meta_client_ip 'IP address of the client that generated the event', +COMMENT COLUMN meta_client_geo_city 'City of the client that generated the event', +COMMENT COLUMN meta_client_geo_country 'Country of the client that generated the event', +COMMENT COLUMN meta_client_geo_country_code 'Country code of the client that generated the event', +COMMENT COLUMN meta_client_geo_continent_code 'Continent code of the client that generated the event', +COMMENT COLUMN meta_client_geo_longitude 'Longitude of the client that generated the event', +COMMENT COLUMN meta_client_geo_latitude 'Latitude of the client that generated the event', +COMMENT COLUMN meta_client_geo_autonomous_system_number 'Autonomous system number of the client that generated the event', +COMMENT COLUMN meta_client_geo_autonomous_system_organization 'Autonomous system organization of the client that generated the event', +COMMENT COLUMN meta_network_id 'Ethereum network ID', +COMMENT COLUMN meta_network_name 'Ethereum network name', +COMMENT COLUMN meta_consensus_version 'Ethereum consensus client version that generated the event', +COMMENT COLUMN meta_consensus_version_major 'Ethereum consensus client major version that generated the event', +COMMENT COLUMN meta_consensus_version_minor 'Ethereum consensus client minor version that generated the event', +COMMENT COLUMN meta_consensus_version_patch 'Ethereum consensus client patch version that generated the event', +COMMENT COLUMN meta_consensus_implementation 'Ethereum consensus client implementation that generated the event', +COMMENT COLUMN meta_labels 'Labels associated with the event'; + +ALTER TABLE default.beacon_api_eth_v1_beacon_committee_local ON CLUSTER '{cluster}' +MODIFY COMMENT 'Contains beacon API /eth/v1/beacon/states/{state_id}/committees data from each sentry client attached to a beacon node.', +COMMENT COLUMN event_date_time 'When the sentry received the event from a beacon node', +COMMENT COLUMN slot 'Slot number in the beacon API committee payload', +COMMENT COLUMN slot_start_date_time 'The wall clock time when the slot started', +COMMENT COLUMN committee_index 'The committee index in the beacon API committee payload', +COMMENT COLUMN validators 'The validator indices in the beacon API committee payload', +COMMENT COLUMN epoch 'The epoch number in the beacon API committee payload', +COMMENT COLUMN epoch_start_date_time 'The wall clock time when the epoch started', +COMMENT COLUMN meta_client_name 'Name of the client that generated the event', +COMMENT COLUMN meta_client_id 'Unique Session ID of the client that generated the event. This changes every time the client is restarted.', +COMMENT COLUMN meta_client_version 'Version of the client that generated the event', +COMMENT COLUMN meta_client_implementation 'Implementation of the client that generated the event', +COMMENT COLUMN meta_client_os 'Operating system of the client that generated the event', +COMMENT COLUMN meta_client_ip 'IP address of the client that generated the event', +COMMENT COLUMN meta_client_geo_city 'City of the client that generated the event', +COMMENT COLUMN meta_client_geo_country 'Country of the client that generated the event', +COMMENT COLUMN meta_client_geo_country_code 'Country code of the client that generated the event', +COMMENT COLUMN meta_client_geo_continent_code 'Continent code of the client that generated the event', +COMMENT COLUMN meta_client_geo_longitude 'Longitude of the client that generated the event', +COMMENT COLUMN meta_client_geo_latitude 'Latitude of the client that generated the event', +COMMENT COLUMN meta_client_geo_autonomous_system_number 'Autonomous system number of the client that generated the event', +COMMENT COLUMN meta_client_geo_autonomous_system_organization 'Autonomous system organization of the client that generated the event', +COMMENT COLUMN meta_network_id 'Ethereum network ID', +COMMENT COLUMN meta_network_name 'Ethereum network name', +COMMENT COLUMN meta_consensus_version 'Ethereum consensus client version that generated the event', +COMMENT COLUMN meta_consensus_version_major 'Ethereum consensus client major version that generated the event', +COMMENT COLUMN meta_consensus_version_minor 'Ethereum consensus client minor version that generated the event', +COMMENT COLUMN meta_consensus_version_patch 'Ethereum consensus client patch version that generated the event', +COMMENT COLUMN meta_consensus_implementation 'Ethereum consensus client implementation that generated the event', +COMMENT COLUMN meta_labels 'Labels associated with the event'; + +ALTER TABLE default.beacon_api_eth_v1_events_blob_sidecar ON CLUSTER '{cluster}' +MODIFY COMMENT 'Contains beacon API eventstream "blob_sidecar" data from each sentry client attached to a beacon node.'; + +ALTER TABLE default.beacon_api_eth_v1_events_blob_sidecar_local ON CLUSTER '{cluster}' +MODIFY COMMENT 'Contains beacon API eventstream "blob_sidecar" data from each sentry client attached to a beacon node.'; + +ALTER TABLE default.beacon_api_eth_v1_validator_attestation_data ON CLUSTER '{cluster}' +MODIFY COMMENT 'Contains beacon API /eth/v1/validator/attestation_data data from each sentry client attached to a beacon node.'; + +ALTER TABLE default.beacon_api_eth_v1_validator_attestation_data_local ON CLUSTER '{cluster}' +MODIFY COMMENT 'Contains beacon API /eth/v1/validator/attestation_data data from each sentry client attached to a beacon node.'; diff --git a/deploy/migrations/clickhouse/028_fix_replacing_tables_sharding_keys.down.sql b/deploy/migrations/clickhouse/029_fix_replacing_tables_sharding_keys.down.sql similarity index 100% rename from deploy/migrations/clickhouse/028_fix_replacing_tables_sharding_keys.down.sql rename to deploy/migrations/clickhouse/029_fix_replacing_tables_sharding_keys.down.sql diff --git a/deploy/migrations/clickhouse/028_fix_replacing_tables_sharding_keys.up.sql b/deploy/migrations/clickhouse/029_fix_replacing_tables_sharding_keys.up.sql similarity index 100% rename from deploy/migrations/clickhouse/028_fix_replacing_tables_sharding_keys.up.sql rename to deploy/migrations/clickhouse/029_fix_replacing_tables_sharding_keys.up.sql