Merge pull request #359 from ethpandaops/feat/clickhouse-beacon-block

feat(deploy): update clickhouse canonical beacon block nullbale columns

deploy/local/docker-compose/vector-kafka-clickhouse.yaml

@@ -1018,19 +1018,21 @@ transforms:
       .proposer_index = .data.message.proposer_index
       .eth1_data_block_hash = .data.message.body.eth1_data.block_hash
       .eth1_data_deposit_root = .data.message.body.eth1_data.deposit_root
-      .execution_payload_block_hash = .data.message.body.execution_payload.block_hash
-      .execution_payload_block_number = .data.message.body.execution_payload.block_number
-      .execution_payload_fee_recipient = .data.message.body.execution_payload.fee_recipient
-      .execution_payload_base_fee_per_gas = .data.message.body.execution_payload.base_fee_per_gas
-      .execution_payload_blob_gas_used = .data.message.body.execution_payload.blob_gas_used
-      .execution_payload_excess_blob_gas = .data.message.body.execution_payload.excess_blob_gas
-      .execution_payload_gas_limit = .data.message.body.execution_payload.gas_limit
-      .execution_payload_gas_used = .data.message.body.execution_payload.gas_used
-      .execution_payload_state_root = .data.message.body.execution_payload.state_root
-      .execution_payload_parent_hash = .data.message.body.execution_payload.parent_hash
-      .execution_payload_transactions_count = .meta.client.additional_data.transactions_count
-      .execution_payload_transactions_total_bytes = .meta.client.additional_data.transactions_total_bytes
-      .execution_payload_transactions_total_bytes_compressed = .meta.client.additional_data.transactions_total_bytes_compressed
+      if exists(.data.message.body.execution_payload) && .data.message.body.execution_payload.block_hash != "" {
+        .execution_payload_block_hash = .data.message.body.execution_payload.block_hash
+        .execution_payload_block_number = .data.message.body.execution_payload.block_number
+        .execution_payload_fee_recipient = .data.message.body.execution_payload.fee_recipient
+        .execution_payload_base_fee_per_gas = .data.message.body.execution_payload.base_fee_per_gas
+        .execution_payload_blob_gas_used = .data.message.body.execution_payload.blob_gas_used
+        .execution_payload_excess_blob_gas = .data.message.body.execution_payload.excess_blob_gas
+        .execution_payload_gas_limit = .data.message.body.execution_payload.gas_limit
+        .execution_payload_gas_used = .data.message.body.execution_payload.gas_used
+        .execution_payload_state_root = .data.message.body.execution_payload.state_root
+        .execution_payload_parent_hash = .data.message.body.execution_payload.parent_hash
+        .execution_payload_transactions_count = .meta.client.additional_data.transactions_count
+        .execution_payload_transactions_total_bytes = .meta.client.additional_data.transactions_total_bytes
+        .execution_payload_transactions_total_bytes_compressed = .meta.client.additional_data.transactions_total_bytes_compressed
+      }
       .updated_date_time = to_unix_timestamp(now())
       del(.event)
       del(.meta)

deploy/migrations/clickhouse/048_nullable_canonical_beacon_block_execution.down.sql

@@ -0,0 +1,13 @@
+ALTER TABLE default.canonical_beacon_block ON CLUSTER '{cluster}'
+    MODIFY COLUMN `execution_payload_block_hash` FixedString(66) COMMENT 'The block hash of the execution payload' CODEC(ZSTD(1)),
+    MODIFY COLUMN `execution_payload_block_number` UInt32 COMMENT 'The block number of the execution payload' CODEC(DoubleDelta, ZSTD(1)),
+    MODIFY COLUMN `execution_payload_fee_recipient` String COMMENT 'The recipient of the fee for this execution payload' CODEC(ZSTD(1)),
+    MODIFY COLUMN `execution_payload_state_root` FixedString(66) COMMENT 'The state root of the execution payload' CODEC(ZSTD(1)),
+    MODIFY COLUMN `execution_payload_parent_hash` FixedString(66) COMMENT 'The parent hash of the execution payload' CODEC(ZSTD(1));
+
+ALTER TABLE default.canonical_beacon_block_local ON CLUSTER '{cluster}'
+    MODIFY COLUMN `execution_payload_block_hash` FixedString(66) COMMENT 'The block hash of the execution payload' CODEC(ZSTD(1)),
+    MODIFY COLUMN `execution_payload_block_number` UInt32 COMMENT 'The block number of the execution payload' CODEC(DoubleDelta, ZSTD(1)),
+    MODIFY COLUMN `execution_payload_fee_recipient` String COMMENT 'The recipient of the fee for this execution payload' CODEC(ZSTD(1)),
+    MODIFY COLUMN `execution_payload_state_root` FixedString(66) COMMENT 'The state root of the execution payload' CODEC(ZSTD(1)),
+    MODIFY COLUMN `execution_payload_parent_hash` FixedString(66) COMMENT 'The parent hash of the execution payload' CODEC(ZSTD(1));

deploy/migrations/clickhouse/048_nullable_canonical_beacon_block_execution.up.sql

@@ -0,0 +1,13 @@
+ALTER TABLE default.canonical_beacon_block_local ON CLUSTER '{cluster}'
+    MODIFY COLUMN `execution_payload_block_hash` Nullable(FixedString(66)) COMMENT 'The block hash of the execution payload' CODEC(ZSTD(1)),
+    MODIFY COLUMN `execution_payload_block_number` Nullable(UInt32) COMMENT 'The block number of the execution payload' CODEC(DoubleDelta, ZSTD(1)),
+    MODIFY COLUMN `execution_payload_fee_recipient` Nullable(String) COMMENT 'The recipient of the fee for this execution payload' CODEC(ZSTD(1)),
+    MODIFY COLUMN `execution_payload_state_root` Nullable(FixedString(66)) COMMENT 'The state root of the execution payload' CODEC(ZSTD(1)),
+    MODIFY COLUMN `execution_payload_parent_hash` Nullable(FixedString(66)) COMMENT 'The parent hash of the execution payload' CODEC(ZSTD(1));
+
+ALTER TABLE default.canonical_beacon_block ON CLUSTER '{cluster}'
+    MODIFY COLUMN `execution_payload_block_hash` Nullable(FixedString(66)) COMMENT 'The block hash of the execution payload' CODEC(ZSTD(1)),
+    MODIFY COLUMN `execution_payload_block_number` Nullable(UInt32) COMMENT 'The block number of the execution payload' CODEC(DoubleDelta, ZSTD(1)),
+    MODIFY COLUMN `execution_payload_fee_recipient` Nullable(String) COMMENT 'The recipient of the fee for this execution payload' CODEC(ZSTD(1)),
+    MODIFY COLUMN `execution_payload_state_root` Nullable(FixedString(66)) COMMENT 'The state root of the execution payload' CODEC(ZSTD(1)),
+    MODIFY COLUMN `execution_payload_parent_hash` Nullable(FixedString(66)) COMMENT 'The parent hash of the execution payload' CODEC(ZSTD(1));