-
Notifications
You must be signed in to change notification settings - Fork 39
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge remote-tracking branch 'origin/feat/logs_new_schema' into feat/…
…logs_new_schema
- Loading branch information
Showing
5 changed files
with
182 additions
and
64 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
9 changes: 9 additions & 0 deletions
9
migrationmanager/migrators/metrics/migrations/000008_samples_agg_tables.down.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,9 @@ | ||
| DROP TABLE IF EXISTS signoz_metrics.samples_v4_agg_5m ON CLUSTER {{.SIGNOZ_CLUSTER}}; | ||
| DROP TABLE IF EXISTS signoz_metrics.samples_v4_agg_5m_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; | ||
| DROP TABLE IF EXISTS signoz_metrics.samples_v4_agg_30m ON CLUSTER {{.SIGNOZ_CLUSTER}}; | ||
| DROP TABLE IF EXISTS signoz_metrics.samples_v4_agg_30m_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; | ||
| DROP TABLE IF EXISTS signoz_metrics.distributed_samples_v4_agg_5m ON CLUSTER {{.SIGNOZ_CLUSTER}}; | ||
| DROP TABLE IF EXISTS signoz_metrics.distributed_samples_v4_agg_30m ON CLUSTER {{.SIGNOZ_CLUSTER}}; | ||
| DROP TABLE IF EXISTS signoz_metrics.time_series_v4_1week ON CLUSTER {{.SIGNOZ_CLUSTER}}; | ||
| DROP TABLE IF EXISTS signoz_metrics.distributed_time_series_v4_1week ON CLUSTER {{.SIGNOZ_CLUSTER}}; | ||
| DROP TABLE IF EXISTS signoz_metrics.time_series_v4_1week_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; |
134 changes: 134 additions & 0 deletions
134
migrationmanager/migrators/metrics/migrations/000008_samples_agg_tables.up.sql
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,134 @@ | ||
| -- 5m aggregation table | ||
| CREATE TABLE IF NOT EXISTS signoz_metrics.samples_v4_agg_5m ON CLUSTER {{.SIGNOZ_CLUSTER}} | ||
| ( | ||
| `env` LowCardinality(String) DEFAULT 'default', | ||
| `temporality` LowCardinality(String) DEFAULT 'Unspecified', | ||
| `metric_name` LowCardinality(String), | ||
| `fingerprint` UInt64 CODEC(ZSTD(1)), | ||
| `unix_milli` Int64 CODEC(DoubleDelta, ZSTD(1)), | ||
| `last` SimpleAggregateFunction(anyLast, Float64) CODEC(ZSTD(1)), | ||
| `min` SimpleAggregateFunction(min, Float64) CODEC(ZSTD(1)), | ||
| `max` SimpleAggregateFunction(max, Float64) CODEC(ZSTD(1)), | ||
| `sum` SimpleAggregateFunction(sum, Float64) CODEC(ZSTD(1)), | ||
| `count` SimpleAggregateFunction(sum, UInt64) CODEC(ZSTD(1)) | ||
| ) | ||
| ENGINE = {{.SIGNOZ_REPLICATED}}AggregatingMergeTree | ||
| PARTITION BY toDate(unix_milli / 1000) | ||
| ORDER BY (env, temporality, metric_name, fingerprint, unix_milli) | ||
| TTL toDateTime(unix_milli/1000) + INTERVAL 2592000 SECOND DELETE | ||
| SETTINGS ttl_only_drop_parts = 1; | ||
|
|
||
| -- 30m aggregation table | ||
| CREATE TABLE IF NOT EXISTS signoz_metrics.samples_v4_agg_30m ON CLUSTER {{.SIGNOZ_CLUSTER}} | ||
| ( | ||
| `env` LowCardinality(String) DEFAULT 'default', | ||
| `temporality` LowCardinality(String) DEFAULT 'Unspecified', | ||
| `metric_name` LowCardinality(String), | ||
| `fingerprint` UInt64 CODEC(ZSTD(1)), | ||
| `unix_milli` Int64 CODEC(DoubleDelta, ZSTD(1)), | ||
| `last` SimpleAggregateFunction(anyLast, Float64) CODEC(ZSTD(1)), | ||
| `min` SimpleAggregateFunction(min, Float64) CODEC(ZSTD(1)), | ||
| `max` SimpleAggregateFunction(max, Float64) CODEC(ZSTD(1)), | ||
| `sum` SimpleAggregateFunction(sum, Float64) CODEC(ZSTD(1)), | ||
| `count` SimpleAggregateFunction(sum, UInt64) CODEC(ZSTD(1)) | ||
| ) | ||
| ENGINE = {{.SIGNOZ_REPLICATED}}AggregatingMergeTree | ||
| PARTITION BY toDate(unix_milli / 1000) | ||
| ORDER BY (env, temporality, metric_name, fingerprint, unix_milli) | ||
| TTL toDateTime(unix_milli/1000) + INTERVAL 2592000 SECOND DELETE | ||
| SETTINGS ttl_only_drop_parts = 1; | ||
|
|
||
| -- 5m aggregation materialized view | ||
| CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_metrics.samples_v4_agg_5m_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} | ||
| TO signoz_metrics.samples_v4_agg_5m AS | ||
| SELECT | ||
| env, | ||
| temporality, | ||
| metric_name, | ||
| fingerprint, | ||
| intDiv(unix_milli, 300000) * 300000 as unix_milli, | ||
| anyLast(value) as last, | ||
| min(value) as min, | ||
| max(value) as max, | ||
| sum(value) as sum, | ||
| count(*) as count | ||
| FROM signoz_metrics.samples_v4 | ||
| GROUP BY | ||
| env, | ||
| temporality, | ||
| metric_name, | ||
| fingerprint, | ||
| unix_milli; | ||
|
|
||
| -- 30m aggregation materialized view | ||
| CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_metrics.samples_v4_agg_30m_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} | ||
| TO signoz_metrics.samples_v4_agg_30m AS | ||
| SELECT | ||
| env, | ||
| temporality, | ||
| metric_name, | ||
| fingerprint, | ||
| intDiv(unix_milli, 1800000) * 1800000 as unix_milli, | ||
| anyLast(last) as last, | ||
| min(min) as min, | ||
| max(max) as max, | ||
| sum(sum) as sum, | ||
| sum(count) as count | ||
| FROM signoz_metrics.samples_v4_agg_5m | ||
| GROUP BY | ||
| env, | ||
| temporality, | ||
| metric_name, | ||
| fingerprint, | ||
| unix_milli; | ||
|
|
||
| -- 5m aggregation distributed table | ||
| CREATE TABLE IF NOT EXISTS signoz_metrics.distributed_samples_v4_agg_5m ON CLUSTER {{.SIGNOZ_CLUSTER}} | ||
| AS signoz_metrics.samples_v4_agg_5m | ||
| ENGINE = Distributed('{{.SIGNOZ_CLUSTER}}', 'signoz_metrics', 'samples_v4_agg_5m', cityHash64(env, temporality, metric_name, fingerprint)); | ||
|
|
||
| -- 30m aggregation distributed table | ||
| CREATE TABLE IF NOT EXISTS signoz_metrics.distributed_samples_v4_agg_30m ON CLUSTER {{.SIGNOZ_CLUSTER}} | ||
| AS signoz_metrics.samples_v4_agg_30m | ||
| ENGINE = Distributed('{{.SIGNOZ_CLUSTER}}', 'signoz_metrics', 'samples_v4_agg_30m', cityHash64(env, temporality, metric_name, fingerprint)); | ||
|
|
||
| -- time series v4 1 week table | ||
| CREATE TABLE IF NOT EXISTS signoz_metrics.time_series_v4_1week ON CLUSTER {{.SIGNOZ_CLUSTER}} ( | ||
| env LowCardinality(String) DEFAULT 'default', | ||
| temporality LowCardinality(String) DEFAULT 'Unspecified', | ||
| metric_name LowCardinality(String), | ||
| description LowCardinality(String) DEFAULT '' CODEC(ZSTD(1)), | ||
| unit LowCardinality(String) DEFAULT '' CODEC(ZSTD(1)), | ||
| type LowCardinality(String) DEFAULT '' CODEC(ZSTD(1)), | ||
| is_monotonic Bool DEFAULT false CODEC(ZSTD(1)), | ||
| fingerprint UInt64 CODEC(Delta, ZSTD), | ||
| unix_milli Int64 CODEC(Delta, ZSTD), | ||
| labels String CODEC(ZSTD(5)), | ||
| INDEX idx_labels labels TYPE ngrambf_v1(4, 1024, 3, 0) GRANULARITY 1 | ||
| ) | ||
| ENGINE = {{.SIGNOZ_REPLICATED}}ReplacingMergeTree | ||
| PARTITION BY toDate(unix_milli / 1000) | ||
| ORDER BY (env, temporality, metric_name, fingerprint, unix_milli) | ||
| TTL toDateTime(unix_milli/1000) + INTERVAL 2592000 SECOND DELETE | ||
| SETTINGS ttl_only_drop_parts = 1; | ||
|
|
||
| -- time series v4 1 week materialized view | ||
| CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_metrics.time_series_v4_1week_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} | ||
| TO signoz_metrics.time_series_v4_1week | ||
| AS SELECT | ||
| env, | ||
| temporality, | ||
| metric_name, | ||
| description, | ||
| unit, | ||
| type, | ||
| is_monotonic, | ||
| fingerprint, | ||
| floor(unix_milli/604800000)*604800000 AS unix_milli, | ||
| labels | ||
| FROM signoz_metrics.time_series_v4_1day; | ||
|
|
||
| -- time series v4 1 week distributed table | ||
| CREATE TABLE IF NOT EXISTS signoz_metrics.distributed_time_series_v4_1week ON CLUSTER {{.SIGNOZ_CLUSTER}} | ||
| AS signoz_metrics.time_series_v4_1week | ||
| ENGINE = Distributed("{{.SIGNOZ_CLUSTER}}", signoz_metrics, time_series_v4_1week, cityHash64(env, temporality, metric_name, fingerprint)); |