Skip to content

Commit

Permalink
Leave only simplifications
Browse files Browse the repository at this point in the history
  • Loading branch information
alexey-milovidov committed Aug 17, 2023
1 parent 857856b commit c3e6f7e
Showing 1 changed file with 6 additions and 8 deletions.
14 changes: 6 additions & 8 deletions docker/test/base/setup_export_logs.sh
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
#!/bin/bash
# shellcheck disable=SC2086

# This script sets up export of system log tables to a remote server.
# Remote tables are created if not exist, and augmented with extra columns,
Expand All @@ -8,7 +7,6 @@

# Pre-configured destination cluster, where to export the data
CLUSTER=${CLUSTER:=system_logs_export}
LOCAL_PARAMETERS=$1

EXTRA_COLUMNS=${EXTRA_COLUMNS:="pull_request_number UInt32, commit_sha String, check_start_time DateTime, check_name LowCardinality(String), instance_type LowCardinality(String), "}
EXTRA_COLUMNS_EXPRESSION=${EXTRA_COLUMNS_EXPRESSION:="0 AS pull_request_number, '' AS commit_sha, now() AS check_start_time, '' AS check_name, '' AS instance_type"}
Expand All @@ -17,21 +15,21 @@ EXTRA_ORDER_BY_COLUMNS=${EXTRA_ORDER_BY_COLUMNS:="check_name, "}
CONNECTION_PARAMETERS=${CONNECTION_PARAMETERS:=""}

# Create all configured system logs:
clickhouse-client $LOCAL_PARAMETERS --query "SYSTEM FLUSH LOGS"
clickhouse-client --query "SYSTEM FLUSH LOGS"

# For each system log table:
clickhouse-client $LOCAL_PARAMETERS --query "SHOW TABLES FROM system LIKE '%\\_log'" | while read -r table
clickhouse-client --query "SHOW TABLES FROM system LIKE '%\\_log'" | while read -r table
do
# Calculate hash of its structure:
hash=$(clickhouse-client $LOCAL_PARAMETERS --query "
hash=$(clickhouse-client --query "
SELECT sipHash64(groupArray((name, type)))
FROM (SELECT name, type FROM system.columns
WHERE database = 'system' AND table = '$table'
ORDER BY position)
")

# Create the destination table with adapted name and structure:
statement=$(clickhouse-client $LOCAL_PARAMETERS --format TSVRaw --query "SHOW CREATE TABLE system.${table}" | sed -r -e '
statement=$(clickhouse-client --format TSVRaw --query "SHOW CREATE TABLE system.${table}" | sed -r -e '
s/^\($/('"$EXTRA_COLUMNS"'/;
s/ORDER BY \(/ORDER BY ('"$EXTRA_ORDER_BY_COLUMNS"'/;
s/^CREATE TABLE system\.\w+_log$/CREATE TABLE IF NOT EXISTS '"$table"'_'"$hash"'/;
Expand All @@ -45,7 +43,7 @@ do
echo "Creating table system.${table}_sender" >&2

# Create Distributed table and materialized view to watch on the original table:
clickhouse-client $LOCAL_PARAMETERS --query "
clickhouse-client --query "
CREATE TABLE system.${table}_sender
ENGINE = Distributed(${CLUSTER}, default, ${table}_${hash})
EMPTY AS
Expand All @@ -55,7 +53,7 @@ do

echo "Creating materialized view system.${table}_watcher" >&2

clickhouse-client $LOCAL_PARAMETERS --query "
clickhouse-client --query "
CREATE MATERIALIZED VIEW system.${table}_watcher TO system.${table}_sender AS
SELECT ${EXTRA_COLUMNS_EXPRESSION}, *
FROM system.${table}
Expand Down

0 comments on commit c3e6f7e

Please sign in to comment.