From 4d5843dc625f55e22872cb50ab0a2fa704743a7a Mon Sep 17 00:00:00 2001 From: Srikanth Chekuri Date: Mon, 16 Sep 2024 15:06:06 +0530 Subject: [PATCH] Add squashed migrations (#397) The goal is to get rid of all `.sql` based operations and files. This changes creates a "squashed" migration containing the current snapshot of the schema --- .../schema_migrator/manager.go | 45 +- .../squashed_logs_migrations.go | 685 +++++++++++ .../squashed_metrics_migrations.go | 925 +++++++++++++++ .../squashed_traces_migrations.go | 1029 +++++++++++++++++ 4 files changed, 2660 insertions(+), 24 deletions(-) create mode 100644 cmd/signozschemamigrator/schema_migrator/squashed_logs_migrations.go create mode 100644 cmd/signozschemamigrator/schema_migrator/squashed_metrics_migrations.go create mode 100644 cmd/signozschemamigrator/schema_migrator/squashed_traces_migrations.go diff --git a/cmd/signozschemamigrator/schema_migrator/manager.go b/cmd/signozschemamigrator/schema_migrator/manager.go index 5cb52bae..0f1ecb7d 100644 --- a/cmd/signozschemamigrator/schema_migrator/manager.go +++ b/cmd/signozschemamigrator/schema_migrator/manager.go @@ -208,14 +208,13 @@ func (m *MigrationManager) runSquashedMigrationsForLogs(ctx context.Context) err return nil } m.logger.Info("Running squashed migrations for logs") - // TODO(srikanthccv): enable in upcoming PR - // for _, migration := range SquashedLogsMigrations { - // for _, item := range migration.UpItems { - // if err := m.RunOperation(ctx, item, migration.MigrationID, "signoz_logs", false); err != nil { - // return err - // } - // } - // } + for _, migration := range SquashedLogsMigrations { + for _, item := range migration.UpItems { + if err := m.RunOperation(ctx, item, migration.MigrationID, "signoz_logs", false); err != nil { + return err + } + } + } m.logger.Info("Squashed migrations for logs completed") return nil } @@ -230,14 +229,13 @@ func (m *MigrationManager) runSquashedMigrationsForMetrics(ctx context.Context) return nil } m.logger.Info("Running squashed migrations for metrics") - // TODO(srikanthccv): enable in upcoming PR - // for _, migration := range SquashedMetricsMigrations { - // for _, item := range migration.UpItems { - // if err := m.RunOperation(ctx, item, migration.MigrationID, signozMetricsDB, false); err != nil { - // return err - // } - // } - // } + for _, migration := range SquashedMetricsMigrations { + for _, item := range migration.UpItems { + if err := m.RunOperation(ctx, item, migration.MigrationID, signozMetricsDB, false); err != nil { + return err + } + } + } m.logger.Info("Squashed migrations for metrics completed") return nil } @@ -252,14 +250,13 @@ func (m *MigrationManager) runSquashedMigrationsForTraces(ctx context.Context) e return nil } m.logger.Info("Running squashed migrations for traces") - // TODO(srikanthccv): enable in upcoming PR - // for _, migration := range SquashedTracesMigrations { - // for _, item := range migration.UpItems { - // if err := m.RunOperation(ctx, item, migration.MigrationID, signozTracesDB, false); err != nil { - // return err - // } - // } - // } + for _, migration := range SquashedTracesMigrations { + for _, item := range migration.UpItems { + if err := m.RunOperation(ctx, item, migration.MigrationID, signozTracesDB, false); err != nil { + return err + } + } + } m.logger.Info("Squashed migrations for traces completed") return nil } diff --git a/cmd/signozschemamigrator/schema_migrator/squashed_logs_migrations.go b/cmd/signozschemamigrator/schema_migrator/squashed_logs_migrations.go new file mode 100644 index 00000000..a2ea8a81 --- /dev/null +++ b/cmd/signozschemamigrator/schema_migrator/squashed_logs_migrations.go @@ -0,0 +1,685 @@ +package schemamigrator + +var ( + SquashedLogsMigrations = []SchemaMigrationRecord{ + { + MigrationID: 1, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_logs", + Table: "logs", + Columns: []Column{ + {Name: "timestamp", Type: ColumnTypeUInt64, Codec: "DoubleDelta, LZ4"}, + {Name: "observed_timestamp", Type: ColumnTypeUInt64, Codec: "DoubleDelta, LZ4"}, + {Name: "id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "trace_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "span_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "trace_flags", Type: ColumnTypeUInt32}, + {Name: "severity_text", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "severity_number", Type: ColumnTypeUInt8}, + {Name: "body", Type: ColumnTypeString, Codec: "ZSTD(2)"}, + {Name: "resources_string_key", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "resources_string_value", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "attributes_string_key", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "attributes_string_value", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "attributes_int64_key", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "attributes_int64_value", Type: ArrayColumnType{ColumnTypeInt64}, Codec: "ZSTD(1)"}, + {Name: "attributes_float64_key", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "attributes_float64_value", Type: ArrayColumnType{ColumnTypeFloat64}, Codec: "ZSTD(1)"}, + {Name: "attributes_bool_key", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "attributes_bool_value", Type: ArrayColumnType{ColumnTypeBool}, Codec: "ZSTD(1)"}, + {Name: "scope_name", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "scope_version", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "scope_string_key", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "scope_string_value", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + }, + Indexes: []Index{ + {Name: "id_minmax", Expression: "id", Type: "minmax", Granularity: 1}, + {Name: "severity_number_idx", Expression: "severity_number", Type: "set(25)", Granularity: 4}, + {Name: "severity_text_idx", Expression: "severity_text", Type: "set(25)", Granularity: 4}, + {Name: "trace_flags_idx", Expression: "trace_flags", Type: "bloom_filter", Granularity: 4}, + {Name: "body_idx", Expression: "lower(body)", Type: "ngrambf_v1(4, 60000, 5, 0)", Granularity: 1}, + {Name: "scope_name_idx", Expression: "scope_name", Type: "tokenbf_v1(10240, 3, 0)", Granularity: 4}, + }, + Engine: MergeTree{ + PartitionBy: "toDate(timestamp / 1000000000)", + OrderBy: "(timestamp, id)", + TTL: "toDateTime(timestamp / 1000000000) + toIntervalSecond(1296000)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "logs", + }, + }, + }, + { + MigrationID: 2, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_logs", + Table: "distributed_logs", + Columns: []Column{ + {Name: "timestamp", Type: ColumnTypeUInt64, Codec: "DoubleDelta, LZ4"}, + {Name: "observed_timestamp", Type: ColumnTypeUInt64, Codec: "DoubleDelta, LZ4"}, + {Name: "id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "trace_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "span_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "trace_flags", Type: ColumnTypeUInt32}, + {Name: "severity_text", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "severity_number", Type: ColumnTypeUInt8}, + {Name: "body", Type: ColumnTypeString, Codec: "ZSTD(2)"}, + {Name: "resources_string_key", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "resources_string_value", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "attributes_string_key", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "attributes_string_value", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "attributes_int64_key", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "attributes_int64_value", Type: ArrayColumnType{ColumnTypeInt64}, Codec: "ZSTD(1)"}, + {Name: "attributes_float64_key", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "attributes_float64_value", Type: ArrayColumnType{ColumnTypeFloat64}, Codec: "ZSTD(1)"}, + {Name: "attributes_bool_key", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "attributes_bool_value", Type: ArrayColumnType{ColumnTypeBool}, Codec: "ZSTD(1)"}, + {Name: "scope_name", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "scope_version", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "scope_string_key", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "scope_string_value", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_logs", + Table: "logs", + ShardingKey: "cityHash64(id)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "distributed_logs", + }, + }, + }, + { + MigrationID: 3, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_logs", + Table: "logs_attribute_keys", + Columns: []Column{ + {Name: "name", Type: ColumnTypeString}, + {Name: "datatype", Type: ColumnTypeString}, + }, + Engine: ReplacingMergeTree{ + MergeTree{ + OrderBy: "(name, datatype)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + }, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "logs_attribute_keys", + }, + }, + }, + { + MigrationID: 4, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_logs", + Table: "distributed_logs_attribute_keys", + Columns: []Column{ + {Name: "name", Type: ColumnTypeString}, + {Name: "datatype", Type: ColumnTypeString}, + }, + Engine: Distributed{ + Database: "signoz_logs", + Table: "logs_attribute_keys", + ShardingKey: "cityHash64(datatype)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "distributed_logs_attribute_keys", + }, + }, + }, + { + MigrationID: 5, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_logs", + ViewName: "attribute_keys_bool_final_mv", + DestTable: "logs_attribute_keys", + Columns: []Column{ + {Name: "name", Type: ColumnTypeString}, + {Name: "datatype", Type: ColumnTypeString}, + }, + Query: `SELECT DISTINCT + arrayJoin(attributes_bool_key) AS name, + 'Bool' AS datatype +FROM signoz_logs.logs +ORDER BY name ASC`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "attribute_keys_bool_final_mv", + }, + }, + }, + { + MigrationID: 6, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_logs", + ViewName: "attribute_keys_float64_final_mv", + DestTable: "logs_attribute_keys", + Columns: []Column{ + {Name: "name", Type: ColumnTypeString}, + {Name: "datatype", Type: ColumnTypeString}, + }, + Query: `SELECT DISTINCT + arrayJoin(attributes_float64_key) AS name, + 'Float64' AS datatype +FROM signoz_logs.logs +ORDER BY name ASC`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "attribute_keys_float64_final_mv", + }, + }, + }, + { + MigrationID: 7, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_logs", + ViewName: "attribute_keys_int64_final_mv", + DestTable: "logs_attribute_keys", + Columns: []Column{ + {Name: "name", Type: ColumnTypeString}, + {Name: "datatype", Type: ColumnTypeString}, + }, + Query: `SELECT DISTINCT + arrayJoin(attributes_int64_key) AS name, + 'Int64' AS datatype +FROM signoz_logs.logs +ORDER BY name ASC`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "attribute_keys_int64_final_mv", + }, + }, + }, + { + MigrationID: 8, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_logs", + ViewName: "attribute_keys_string_final_mv", + DestTable: "logs_attribute_keys", + Columns: []Column{ + {Name: "name", Type: ColumnTypeString}, + {Name: "datatype", Type: ColumnTypeString}, + }, + Query: `SELECT DISTINCT + arrayJoin(attributes_string_key) AS name, + 'String' AS datatype +FROM signoz_logs.logs +ORDER BY name ASC`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "attribute_keys_string_final_mv", + }, + }, + }, + { + MigrationID: 9, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_logs", + Table: "logs_resource_keys", + Columns: []Column{ + {Name: "name", Type: ColumnTypeString}, + {Name: "datatype", Type: ColumnTypeString}, + }, + Engine: ReplacingMergeTree{ + MergeTree{ + OrderBy: "(name, datatype)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + }, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "logs_resource_keys", + }, + }, + }, + { + MigrationID: 10, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_logs", + Table: "distributed_logs_resource_keys", + Columns: []Column{ + {Name: "name", Type: ColumnTypeString}, + {Name: "datatype", Type: ColumnTypeString}, + }, + Engine: Distributed{ + Database: "signoz_logs", + Table: "logs_resource_keys", + ShardingKey: "cityHash64(datatype)", + }, + }, + }, + }, + { + MigrationID: 11, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_logs", + ViewName: "resource_keys_string_final_mv", + DestTable: "logs_resource_keys", + Columns: []Column{ + {Name: "name", Type: ColumnTypeString}, + {Name: "datatype", Type: ColumnTypeString}, + }, + Query: `SELECT DISTINCT + arrayJoin(resources_string_key) AS name, + 'String' AS datatype +FROM signoz_logs.logs +ORDER BY name ASC`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "resource_keys_string_final_mv", + }, + }, + }, + { + MigrationID: 12, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_logs", + Table: "tag_attributes", + Columns: []Column{ + {Name: "timestamp", Type: DateTimeColumnType{}, Codec: "ZSTD(1)"}, + {Name: "tagKey", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "tagType", Type: EnumerationColumnType{Values: []string{"'tag' = 1", "'resource' = 2", "'scope' = 3"}, Size: 8}, Codec: "ZSTD(1)"}, + {Name: "tagDataType", Type: EnumerationColumnType{Values: []string{"'string' = 1", "'bool' = 2", "'int64' = 3", "'float64' = 4"}, Size: 8}, Codec: "ZSTD(1)"}, + {Name: "stringTagValue", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "int64TagValue", Type: NullableColumnType{ColumnTypeInt64}, Codec: "ZSTD(1)"}, + {Name: "float64TagValue", Type: NullableColumnType{ColumnTypeFloat64}, Codec: "ZSTD(1)"}, + }, + Engine: ReplacingMergeTree{ + MergeTree: MergeTree{ + OrderBy: "(tagKey, tagType, tagDataType, stringTagValue, int64TagValue, float64TagValue)", + TTL: "toDateTime(timestamp) + toIntervalSecond(172800)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + {Name: "allow_nullable_key", Value: "1"}, + }, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "tag_attributes", + }, + }, + }, + { + MigrationID: 13, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_logs", + Table: "distributed_tag_attributes", + Columns: []Column{ + {Name: "timestamp", Type: DateTimeColumnType{}, Codec: "ZSTD(1)"}, + {Name: "tagKey", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "tagType", Type: EnumerationColumnType{Values: []string{"'tag' = 1", "'resource' = 2", "'scope' = 3"}, Size: 8}, Codec: "ZSTD(1)"}, + {Name: "tagDataType", Type: EnumerationColumnType{Values: []string{"'string' = 1", "'bool' = 2", "'int64' = 3", "'float64' = 4"}, Size: 8}, Codec: "ZSTD(1)"}, + {Name: "stringTagValue", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "int64TagValue", Type: NullableColumnType{ColumnTypeInt64}, Codec: "ZSTD(1)"}, + {Name: "float64TagValue", Type: NullableColumnType{ColumnTypeFloat64}, Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_logs", + Table: "tag_attributes", + ShardingKey: "rand()", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "distributed_tag_attributes", + }, + }, + }, + { + MigrationID: 14, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_logs", + Table: "usage", + Columns: []Column{ + {Name: "tenant", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "collector_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "exporter_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "timestamp", Type: DateTimeColumnType{}, Codec: "ZSTD(1)"}, + {Name: "data", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + }, + Engine: MergeTree{ + OrderBy: "(tenant, collector_id, exporter_id, timestamp)", + TTL: "timestamp + toIntervalDay(3)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "usage", + }, + }, + }, + { + MigrationID: 15, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_logs", + Table: "distributed_usage", + Columns: []Column{ + {Name: "tenant", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "collector_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "exporter_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "timestamp", Type: DateTimeColumnType{}, Codec: "ZSTD(1)"}, + {Name: "data", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_logs", + Table: "usage", + ShardingKey: "cityHash64(rand())", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "distributed_usage", + }, + }, + }, + { + MigrationID: 16, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_logs", + Table: "logs_v2", + Columns: []Column{ + {Name: "ts_bucket_start", Type: ColumnTypeUInt64, Codec: "DoubleDelta, LZ4"}, + {Name: "resource_fingerprint", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "timestamp", Type: ColumnTypeUInt64, Codec: "DoubleDelta, LZ4"}, + {Name: "observed_timestamp", Type: ColumnTypeUInt64, Codec: "DoubleDelta, LZ4"}, + {Name: "id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "trace_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "span_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "trace_flags", Type: ColumnTypeUInt32}, + {Name: "severity_text", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "severity_number", Type: ColumnTypeUInt8}, + {Name: "body", Type: ColumnTypeString, Codec: "ZSTD(2)"}, + {Name: "attributes_string", Type: MapColumnType{LowCardinalityColumnType{ColumnTypeString}, ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "attributes_number", Type: MapColumnType{LowCardinalityColumnType{ColumnTypeString}, ColumnTypeFloat64}, Codec: "ZSTD(1)"}, + {Name: "attributes_bool", Type: MapColumnType{LowCardinalityColumnType{ColumnTypeString}, ColumnTypeBool}, Codec: "ZSTD(1)"}, + {Name: "resource_string", Type: MapColumnType{LowCardinalityColumnType{ColumnTypeString}, ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "scope_name", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "scope_version", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "scope_string", Type: MapColumnType{LowCardinalityColumnType{ColumnTypeString}, ColumnTypeString}, Codec: "ZSTD(1)"}, + }, + Indexes: []Index{ + {Name: "id_minmax", Expression: "id", Type: "minmax", Granularity: 1}, + {Name: "severity_number_idx", Expression: "severity_number", Type: "set(25)", Granularity: 4}, + {Name: "severity_text_idx", Expression: "severity_text", Type: "set(25)", Granularity: 4}, + {Name: "trace_flags_idx", Expression: "trace_flags", Type: "bloom_filter", Granularity: 4}, + {Name: "body_idx", Expression: "lower(body)", Type: "ngrambf_v1(4, 60000, 5, 0)", Granularity: 1}, + {Name: "scope_name_idx", Expression: "scope_name", Type: "tokenbf_v1(10240, 3, 0)", Granularity: 4}, + {Name: "attributes_string_idx_key", Expression: "mapKeys(attributes_string)", Type: "tokenbf_v1(1024, 2, 0)", Granularity: 1}, + {Name: "attributes_string_idx_val", Expression: "mapValues(attributes_string)", Type: "ngrambf_v1(4, 5000, 2, 0)", Granularity: 1}, + {Name: "attributes_number_idx_key", Expression: "mapKeys(attributes_number)", Type: "tokenbf_v1(1024, 2, 0)", Granularity: 1}, + {Name: "attributes_number_idx_val", Expression: "mapValues(attributes_number)", Type: "bloom_filter", Granularity: 1}, + {Name: "attributes_bool_idx_key", Expression: "mapKeys(attributes_bool)", Type: "tokenbf_v1(1024, 2, 0)", Granularity: 1}, + }, + Engine: MergeTree{ + PartitionBy: "toDate(timestamp / 1000000000)", + OrderBy: "(ts_bucket_start, resource_fingerprint, severity_text, timestamp, id)", + TTL: "toDateTime(timestamp / 1000000000) + toIntervalSecond(1296000)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "logs_v2", + }, + }, + }, + { + MigrationID: 17, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_logs", + Table: "distributed_logs_v2", + Columns: []Column{ + {Name: "ts_bucket_start", Type: ColumnTypeUInt64, Codec: "DoubleDelta, LZ4"}, + {Name: "resource_fingerprint", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "timestamp", Type: ColumnTypeUInt64, Codec: "DoubleDelta, LZ4"}, + {Name: "observed_timestamp", Type: ColumnTypeUInt64, Codec: "DoubleDelta, LZ4"}, + {Name: "id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "trace_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "span_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "trace_flags", Type: ColumnTypeUInt32}, + {Name: "severity_text", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "severity_number", Type: ColumnTypeUInt8}, + {Name: "body", Type: ColumnTypeString, Codec: "ZSTD(2)"}, + {Name: "attributes_string", Type: MapColumnType{LowCardinalityColumnType{ColumnTypeString}, ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "attributes_number", Type: MapColumnType{LowCardinalityColumnType{ColumnTypeString}, ColumnTypeFloat64}, Codec: "ZSTD(1)"}, + {Name: "attributes_bool", Type: MapColumnType{LowCardinalityColumnType{ColumnTypeString}, ColumnTypeBool}, Codec: "ZSTD(1)"}, + {Name: "resource_string", Type: MapColumnType{LowCardinalityColumnType{ColumnTypeString}, ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "scope_name", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "scope_version", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "scope_string", Type: MapColumnType{LowCardinalityColumnType{ColumnTypeString}, ColumnTypeString}, Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_logs", + Table: "logs_v2", + ShardingKey: "cityHash64(id)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "distributed_logs_v2", + }, + }, + }, + { + MigrationID: 18, + UpItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "resource_keys_string_final_mv", + }, + DropTableOperation{ + Database: "signoz_logs", + Table: "attribute_keys_float64_final_mv", + }, + DropTableOperation{ + Database: "signoz_logs", + Table: "attribute_keys_int64_final_mv", + }, + DropTableOperation{ + Database: "signoz_logs", + Table: "attribute_keys_string_final_mv", + }, + DropTableOperation{ + Database: "signoz_logs", + Table: "attribute_keys_bool_final_mv", + }, + }, + }, + { + MigrationID: 19, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_logs", + ViewName: "attribute_keys_string_final_mv", + DestTable: "logs_attribute_keys", + Columns: []Column{ + {Name: "name", Type: ColumnTypeString}, + {Name: "datatype", Type: ColumnTypeString}, + }, + Query: `SELECT DISTINCT + arrayJoin(mapKeys(attributes_string)) AS name, + 'String' AS datatype +FROM signoz_logs.logs_v2 +ORDER BY name ASC`, + }, + CreateMaterializedViewOperation{ + Database: "signoz_logs", + ViewName: "attribute_keys_float64_final_mv", + DestTable: "logs_attribute_keys", + Columns: []Column{ + {Name: "name", Type: ColumnTypeString}, + {Name: "datatype", Type: ColumnTypeString}, + }, + Query: `SELECT DISTINCT + arrayJoin(mapKeys(attributes_number)) AS name, + 'Float64' AS datatype +FROM signoz_logs.logs_v2 +ORDER BY name ASC`, + }, + CreateMaterializedViewOperation{ + Database: "signoz_logs", + ViewName: "attribute_keys_bool_final_mv", + DestTable: "logs_attribute_keys", + Columns: []Column{ + {Name: "name", Type: ColumnTypeString}, + {Name: "datatype", Type: ColumnTypeString}, + }, + Query: `SELECT DISTINCT + arrayJoin(mapKeys(attributes_bool)) AS name, + 'Bool' AS datatype +FROM signoz_logs.logs_v2 +ORDER BY name ASC`, + }, + CreateMaterializedViewOperation{ + Database: "signoz_logs", + ViewName: "resource_keys_string_final_mv", + DestTable: "logs_resource_keys", + Columns: []Column{ + {Name: "name", Type: ColumnTypeString}, + {Name: "datatype", Type: ColumnTypeString}, + }, + Query: `SELECT DISTINCT + arrayJoin(mapKeys(resources_string)) AS name, + 'String' AS datatype +FROM signoz_logs.logs_v2 +ORDER BY name ASC`, + }, + }, + }, + { + MigrationID: 20, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_logs", + Table: "logs_v2_resource", + Columns: []Column{ + {Name: "labels", Type: ColumnTypeString, Codec: "ZSTD(5)"}, + {Name: "fingerprint", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "seen_at_ts_bucket_start", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + }, + Indexes: []Index{ + {Name: "idx_labels", Expression: "lower(labels)", Type: "ngrambf_v1(4, 1024, 3, 0)", Granularity: 1}, + {Name: "idx_labels_v1", Expression: "labels", Type: "ngrambf_v1(4, 1024, 3, 0)", Granularity: 1}, + }, + Engine: ReplacingMergeTree{ + MergeTree: MergeTree{ + PartitionBy: "toDate(seen_at_ts_bucket_start / 1000)", + OrderBy: "(labels, fingerprint, seen_at_ts_bucket_start)", + TTL: "toDateTime(seen_at_ts_bucket_start) + INTERVAL 1296000 SECOND + INTERVAL 1800 SECOND DELETE", + Settings: TableSettings{ + {Name: "ttl_only_drop_parts", Value: "1"}, + {Name: "index_granularity", Value: "8192"}, + }, + }, + }, + }, + CreateTableOperation{ + Database: "signoz_logs", + Table: "distributed_logs_v2_resource", + Columns: []Column{ + {Name: "labels", Type: ColumnTypeString, Codec: "ZSTD(5)"}, + {Name: "fingerprint", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "seen_at_ts_bucket_start", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_logs", + Table: "logs_v2_resource", + ShardingKey: "cityHash64(labels, fingerprint)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_logs", + Table: "logs_v2_resource", + }, + DropTableOperation{ + Database: "signoz_logs", + Table: "distributed_logs_v2_resource", + }, + }, + }, + } +) diff --git a/cmd/signozschemamigrator/schema_migrator/squashed_metrics_migrations.go b/cmd/signozschemamigrator/schema_migrator/squashed_metrics_migrations.go new file mode 100644 index 00000000..351fe5d5 --- /dev/null +++ b/cmd/signozschemamigrator/schema_migrator/squashed_metrics_migrations.go @@ -0,0 +1,925 @@ +package schemamigrator + +var ( + SquashedMetricsMigrations = []SchemaMigrationRecord{ + { + MigrationID: 1, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "samples_v2", + Columns: []Column{ + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "ZSTD(1)"}, + {Name: "timestamp_ms", Type: ColumnTypeInt64, Codec: "DoubleDelta, LZ4"}, + {Name: "value", Type: ColumnTypeFloat64, Codec: "Gorilla, LZ4"}, + }, + Engine: MergeTree{ + PartitionBy: "toDate(timestamp_ms / 1000)", + OrderBy: "(metric_name, fingerprint, timestamp_ms)", + TTL: "toDateTime(timestamp_ms / 1000) + toIntervalSecond(2592000)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "samples_v2", + }, + }, + }, + { + MigrationID: 2, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "distributed_samples_v2", + Columns: []Column{ + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "ZSTD(1)"}, + {Name: "timestamp_ms", Type: ColumnTypeInt64, Codec: "DoubleDelta, LZ4"}, + {Name: "value", Type: ColumnTypeFloat64, Codec: "Gorilla, LZ4"}, + }, + Engine: Distributed{ + Database: "signoz_metrics", + Table: "samples_v2", + ShardingKey: "cityHash64(metric_name, fingerprint)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "distributed_samples_v2", + }, + }, + }, + { + MigrationID: 3, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v2", + Columns: []Column{ + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "ZSTD(1)"}, + {Name: "timestamp_ms", Type: ColumnTypeInt64, Codec: "DoubleDelta, LZ4"}, + {Name: "labels", Type: ColumnTypeString, Codec: "ZSTD(5)"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'", Codec: "ZSTD(5)"}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + }, + Indexes: []Index{ + {Name: "temporality_index", Expression: "temporality", Type: "SET(3)", Granularity: 1}, + }, + Engine: ReplacingMergeTree{ + MergeTree{ + PartitionBy: "toDate(timestamp_ms / 1000)", + OrderBy: "(metric_name, fingerprint)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + }, + }, + }, + }, + }, + DownItems: []Operation{}, + }, + { + MigrationID: 4, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v2", + Columns: []Column{ + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "ZSTD(1)"}, + {Name: "timestamp_ms", Type: ColumnTypeInt64, Codec: "DoubleDelta, LZ4"}, + {Name: "labels", Type: ColumnTypeString, Codec: "ZSTD(5)"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'", Codec: "ZSTD(5)"}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_metrics", + Table: "time_series_v2", + ShardingKey: "cityHash64(metric_name, fingerprint)", + }, + }, + }, + DownItems: []Operation{}, + }, + { + MigrationID: 5, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "exp_hist", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "DoubleDelta, ZSTD(1)"}, + {Name: "count", Type: ColumnTypeUInt64, Codec: "ZSTD(1)"}, + {Name: "sum", Type: ColumnTypeFloat64, Codec: "Gorilla, ZSTD(1)"}, + {Name: "min", Type: ColumnTypeFloat64, Codec: "Gorilla, ZSTD(1)"}, + {Name: "max", Type: ColumnTypeFloat64, Codec: "Gorilla, ZSTD(1)"}, + {Name: "sketch", Type: AggregateFunction{FunctionName: "quantilesDD(0.01, 0.5, 0.75, 0.9, 0.95, 0.99)", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "ZSTD(1)"}, + }, + Engine: MergeTree{ + PartitionBy: "toDate(unix_milli / 1000)", + OrderBy: "(env, temporality, metric_name, fingerprint, unix_milli)", + TTL: "toDateTime(unix_milli / 1000) + toIntervalSecond(2592000)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "exp_hist", + }, + }, + }, + { + MigrationID: 6, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "distributed_exp_hist", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "DoubleDelta, ZSTD(1)"}, + {Name: "count", Type: ColumnTypeUInt64, Codec: "ZSTD(1)"}, + {Name: "sum", Type: ColumnTypeFloat64, Codec: "Gorilla, ZSTD(1)"}, + {Name: "min", Type: ColumnTypeFloat64, Codec: "Gorilla, ZSTD(1)"}, + {Name: "max", Type: ColumnTypeFloat64, Codec: "Gorilla, ZSTD(1)"}, + {Name: "sketch", Type: AggregateFunction{FunctionName: "quantilesDD(0.01, 0.5, 0.75, 0.9, 0.95, 0.99)", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_metrics", + Table: "exp_hist", + ShardingKey: "cityHash64(env, temporality, metric_name, fingerprint)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "distributed_exp_hist", + }, + }, + }, + { + MigrationID: 7, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "samples_v4", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "DoubleDelta, ZSTD(1)"}, + {Name: "value", Type: ColumnTypeFloat64, Codec: "Gorilla, ZSTD(1)"}, + }, + Engine: MergeTree{ + PartitionBy: "toDate(unix_milli / 1000)", + OrderBy: "(env, temporality, metric_name, fingerprint, unix_milli)", + TTL: "toDateTime(unix_milli / 1000) + toIntervalSecond(2592000)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "samples_v4", + }, + }, + }, + { + MigrationID: 8, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "distributed_samples_v4", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "DoubleDelta, ZSTD(1)"}, + {Name: "value", Type: ColumnTypeFloat64, Codec: "Gorilla, ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_metrics", + Table: "samples_v4", + ShardingKey: "cityHash64(env, temporality, metric_name, fingerprint)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "distributed_samples_v4", + }, + }, + }, + { + MigrationID: 9, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Codec: "ZSTD(5)"}, + }, + Indexes: []Index{ + {Name: "idx_labels", Expression: "labels", Type: "ngrambf_v1(4, 1024, 3, 0)", Granularity: 1}, + }, + Engine: ReplacingMergeTree{ + MergeTree{ + PartitionBy: "toDate(unix_milli / 1000)", + OrderBy: "(env, temporality, metric_name, fingerprint, unix_milli)", + TTL: "toDateTime(unix_milli / 1000) + toIntervalSecond(2592000)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4", + }, + }, + }, + { + MigrationID: 10, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v4", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Codec: "ZSTD(5)"}, + }, + Engine: Distributed{ + Database: "signoz_metrics", + Table: "time_series_v4", + ShardingKey: "cityHash64(env, temporality, metric_name, fingerprint)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v4", + }, + }, + }, + { + MigrationID: 11, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_6hrs", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Codec: "ZSTD(5)"}, + }, + Indexes: []Index{ + {Name: "idx_labels", Expression: "labels", Type: "ngrambf_v1(4, 1024, 3, 0)", Granularity: 1}, + }, + Engine: ReplacingMergeTree{ + MergeTree{ + PartitionBy: "toDate(unix_milli / 1000)", + OrderBy: "(env, temporality, metric_name, fingerprint, unix_milli)", + TTL: "toDateTime(unix_milli / 1000) + toIntervalSecond(2592000)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_6hrs", + }, + }, + }, + { + MigrationID: 12, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v4_6hrs", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Codec: "ZSTD(5)"}, + }, + Engine: Distributed{ + Database: "signoz_metrics", + Table: "time_series_v4", + ShardingKey: "cityHash64(env, temporality, metric_name, fingerprint)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v4_6hrs", + }, + }, + }, + { + MigrationID: 13, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_1day", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Codec: "ZSTD(5)"}, + }, + Indexes: []Index{ + {Name: "idx_labels", Expression: "labels", Type: "ngrambf_v1(4, 1024, 3, 0)", Granularity: 1}, + }, + Engine: ReplacingMergeTree{ + MergeTree{ + PartitionBy: "toDate(unix_milli / 1000)", + OrderBy: "(env, temporality, metric_name, fingerprint, unix_milli)", + TTL: "toDateTime(unix_milli / 1000) + toIntervalSecond(2592000)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_1day", + }, + }, + }, + { + MigrationID: 14, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v4_1day", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Codec: "ZSTD(5)"}, + }, + Engine: Distributed{ + Database: "signoz_metrics", + Table: "time_series_v4_1day", + ShardingKey: "cityHash64(env, temporality, metric_name, fingerprint)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v4_1day", + }, + }, + }, + { + MigrationID: 15, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_1week", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Codec: "ZSTD(5)"}, + }, + Indexes: []Index{ + {Name: "idx_labels", Expression: "labels", Type: "ngrambf_v1(4, 1024, 3, 0)", Granularity: 1}, + }, + Engine: ReplacingMergeTree{ + MergeTree{ + PartitionBy: "toDate(unix_milli / 1000)", + OrderBy: "(env, temporality, metric_name, fingerprint, unix_milli)", + TTL: "toDateTime(unix_milli / 1000) + toIntervalSecond(2592000)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_1week", + }, + }, + }, + { + MigrationID: 16, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v4_1week", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Codec: "ZSTD(5)"}, + }, + Engine: Distributed{ + Database: "signoz_metrics", + Table: "time_series_v4_1week", + ShardingKey: "cityHash64(env, temporality, metric_name, fingerprint)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "distributed_time_series_v4_1week", + }, + }, + }, + { + MigrationID: 17, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_metrics", + ViewName: "time_series_v4_6hrs_mv", + DestTable: "time_series_v4_6hrs", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Codec: "ZSTD(5)"}, + }, + Query: `SELECT + env, + temporality, + metric_name, + description, + unit, + type, + is_monotonic, + fingerprint, + floor(unix_milli / 21600000) * 21600000 AS unix_milli, + labels +FROM signoz_metrics.time_series_v4`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_6hrs_mv", + }, + }, + }, + { + MigrationID: 18, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_metrics", + ViewName: "time_series_v4_1day_mv", + DestTable: "time_series_v4_1day", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Codec: "ZSTD(5)"}, + }, + Query: `SELECT + env, + temporality, + metric_name, + description, + unit, + type, + is_monotonic, + fingerprint, + floor(unix_milli / 86400000) * 86400000 AS unix_milli, + labels +FROM signoz_metrics.time_series_v4`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_1day_mv", + }, + }, + }, + { + MigrationID: 19, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_metrics", + ViewName: "time_series_v4_1week_mv", + DestTable: "time_series_v4_1week", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "description", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "unit", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "type", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "''", Codec: "ZSTD(1)"}, + {Name: "is_monotonic", Type: ColumnTypeBool, Default: "false", Codec: "ZSTD(1)"}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "labels", Type: ColumnTypeString, Codec: "ZSTD(5)"}, + }, + Query: `SELECT + env, + temporality, + metric_name, + description, + unit, + type, + is_monotonic, + fingerprint, + floor(unix_milli / 604800000) * 604800000 AS unix_milli, + labels +FROM signoz_metrics.time_series_v4_1day`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "time_series_v4_1week_mv", + }, + }, + }, + { + MigrationID: 20, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "samples_v4_agg_5m", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "last", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"}, + {Name: "min", Type: SimpleAggregateFunction{FunctionName: "min", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"}, + {Name: "max", Type: SimpleAggregateFunction{FunctionName: "max", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"}, + {Name: "sum", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"}, + {Name: "count", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "ZSTD(1)"}, + }, + Engine: AggregatingMergeTree{ + MergeTree{ + PartitionBy: "toDate(unix_milli / 1000)", + OrderBy: "(env, temporality, metric_name, fingerprint, unix_milli)", + TTL: "toDateTime(unix_milli / 1000) + toIntervalSecond(2592000)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "samples_v4_agg_5m", + }, + }, + }, + { + MigrationID: 21, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "samples_v4_agg_30m", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "last", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"}, + {Name: "min", Type: SimpleAggregateFunction{FunctionName: "min", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"}, + {Name: "max", Type: SimpleAggregateFunction{FunctionName: "max", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"}, + {Name: "sum", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"}, + {Name: "count", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "ZSTD(1)"}, + }, + Engine: AggregatingMergeTree{ + MergeTree{ + PartitionBy: "toDate(unix_milli / 1000)", + OrderBy: "(env, temporality, metric_name, fingerprint, unix_milli)", + TTL: "toDateTime(unix_milli / 1000) + toIntervalSecond(2592000)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "samples_v4_agg_30m", + }, + }, + }, + { + MigrationID: 22, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_metrics", + ViewName: "samples_v4_agg_5m_mv", + DestTable: "samples_v4_agg_5m", + Query: `SELECT + env, + temporality, + metric_name, + fingerprint, + intDiv(unix_milli, 300000) * 300000 as unix_milli, + anyLast(value) as last, + min(value) as min, + max(value) as max, + sum(value) as sum, + count(*) as count +FROM signoz_metrics.samples_v4 +GROUP BY + env, + temporality, + metric_name, + fingerprint, + unix_milli;`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "samples_v4_agg_5m_mv", + }, + }, + }, + { + MigrationID: 23, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_metrics", + ViewName: "samples_v4_agg_30m_mv", + DestTable: "samples_v4_agg_30m", + Query: `SELECT + env, + temporality, + metric_name, + fingerprint, + intDiv(unix_milli, 1800000) * 1800000 as unix_milli, + anyLast(last) as last, + min(min) as min, + max(max) as max, + sum(sum) as sum, + sum(count) as count +FROM signoz_metrics.samples_v4_agg_5m +GROUP BY + env, + temporality, + metric_name, + fingerprint, + unix_milli;`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "samples_v4_agg_30m_mv", + }, + }, + }, + { + MigrationID: 24, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "distributed_samples_v4_agg_5m", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "last", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"}, + {Name: "min", Type: SimpleAggregateFunction{FunctionName: "min", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"}, + {Name: "max", Type: SimpleAggregateFunction{FunctionName: "max", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"}, + {Name: "sum", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"}, + {Name: "count", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_metrics", + Table: "samples_v4_agg_5m", + ShardingKey: "cityHash64(env, temporality, metric_name, fingerprint)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "distributed_samples_v4_agg_5m", + }, + }, + }, + { + MigrationID: 25, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "distributed_samples_v4_agg_30m", + Columns: []Column{ + {Name: "env", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'default'"}, + {Name: "temporality", Type: LowCardinalityColumnType{ColumnTypeString}, Default: "'Unspecified'"}, + {Name: "metric_name", Type: LowCardinalityColumnType{ColumnTypeString}}, + {Name: "fingerprint", Type: ColumnTypeUInt64, Codec: "ZSTD(1)"}, + {Name: "unix_milli", Type: ColumnTypeInt64, Codec: "Delta(8), ZSTD(1)"}, + {Name: "last", Type: SimpleAggregateFunction{FunctionName: "anyLast", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"}, + {Name: "min", Type: SimpleAggregateFunction{FunctionName: "min", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"}, + {Name: "max", Type: SimpleAggregateFunction{FunctionName: "max", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"}, + {Name: "sum", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"}, + {Name: "count", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_metrics", + Table: "samples_v4_agg_30m", + ShardingKey: "cityHash64(env, temporality, metric_name, fingerprint)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "distributed_samples_v4_agg_30m", + }, + }, + }, + { + MigrationID: 26, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "usage", + Columns: []Column{ + {Name: "tenant", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "collector_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "exporter_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "timestamp", Type: DateTimeColumnType{}, Codec: "ZSTD(1)"}, + {Name: "data", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + }, + Engine: MergeTree{ + OrderBy: "(tenant, collector_id, exporter_id, timestamp)", + TTL: "timestamp + toIntervalDay(3)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "usage", + }, + }, + }, + { + MigrationID: 27, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_metrics", + Table: "distributed_usage", + Columns: []Column{ + {Name: "tenant", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "collector_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "exporter_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "timestamp", Type: DateTimeColumnType{}, Codec: "ZSTD(1)"}, + {Name: "data", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_metrics", + Table: "usage", + ShardingKey: "cityHash64(rand())", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_metrics", + Table: "distributed_usage", + }, + }, + }, + } +) diff --git a/cmd/signozschemamigrator/schema_migrator/squashed_traces_migrations.go b/cmd/signozschemamigrator/schema_migrator/squashed_traces_migrations.go new file mode 100644 index 00000000..3db80fdb --- /dev/null +++ b/cmd/signozschemamigrator/schema_migrator/squashed_traces_migrations.go @@ -0,0 +1,1029 @@ +package schemamigrator + +var ( + SquashedTracesMigrations = []SchemaMigrationRecord{ + { + MigrationID: 1, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "signoz_index_v2", + Columns: []Column{ + {Name: "timestamp", Type: DateTime64ColumnType{Precision: 9}, Codec: "DoubleDelta, LZ4"}, + {Name: "traceID", Type: FixedStringColumnType{Length: 32}, Codec: "ZSTD(1)"}, + {Name: "spanID", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "parentSpanID", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "serviceName", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "kind", Type: ColumnTypeInt8, Codec: "T64, ZSTD(1)"}, + {Name: "durationNano", Type: ColumnTypeUInt64, Codec: "T64, ZSTD(1)"}, + {Name: "statusCode", Type: ColumnTypeInt16, Codec: "T64, ZSTD(1)"}, + {Name: "externalHttpMethod", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "externalHttpUrl", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "dbSystem", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "dbName", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "dbOperation", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "peerService", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "events", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(2)"}, + {Name: "httpMethod", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "httpUrl", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "httpRoute", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "httpHost", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "msgSystem", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "msgOperation", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "hasError", Type: ColumnTypeBool, Codec: "T64, ZSTD(1)"}, + {Name: "rpcSystem", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "rpcService", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "rpcMethod", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "responseStatusCode", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "stringTagMap", Type: MapColumnType{ColumnTypeString, ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "numberTagMap", Type: MapColumnType{ColumnTypeString, ColumnTypeFloat64}, Codec: "ZSTD(1)"}, + {Name: "boolTagMap", Type: MapColumnType{ColumnTypeString, ColumnTypeBool}, Codec: "ZSTD(1)"}, + {Name: "resourceTagsMap", Type: MapColumnType{LowCardinalityColumnType{ColumnTypeString}, ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "isRemote", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "statusMessage", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "statusCodeString", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "spanKind", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + }, + Indexes: []Index{ + {Name: "idx_service", Expression: "serviceName", Type: "bloom_filter", Granularity: 4}, + {Name: "idx_name", Expression: "name", Type: "bloom_filter", Granularity: 4}, + {Name: "idx_kind", Expression: "kind", Type: "minmax", Granularity: 4}, + {Name: "idx_duration", Expression: "durationNano", Type: "minmax", Granularity: 1}, + {Name: "idx_hasError", Expression: "hasError", Type: "set(2)", Granularity: 1}, + {Name: "idx_httpRoute", Expression: "httpRoute", Type: "bloom_filter", Granularity: 4}, + {Name: "idx_httpUrl", Expression: "httpUrl", Type: "bloom_filter", Granularity: 4}, + {Name: "idx_httpHost", Expression: "httpHost", Type: "bloom_filter", Granularity: 4}, + {Name: "idx_httpMethod", Expression: "httpMethod", Type: "bloom_filter", Granularity: 4}, + {Name: "idx_timestamp", Expression: "timestamp", Type: "minmax", Granularity: 1}, + {Name: "idx_rpcMethod", Expression: "rpcMethod", Type: "bloom_filter", Granularity: 4}, + {Name: "idx_responseStatusCode", Expression: "responseStatusCode", Type: "set(0)", Granularity: 1}, + {Name: "idx_resourceTagsMapKeys", Expression: "mapKeys(resourceTagsMap)", Type: "bloom_filter(0.01)", Granularity: 64}, + {Name: "idx_resourceTagsMapValues", Expression: "mapValues(resourceTagsMap)", Type: "bloom_filter(0.01)", Granularity: 64}, + {Name: "idx_statusCodeString", Expression: "statusCodeString", Type: "set(3)", Granularity: 4}, + {Name: "idx_spanKind", Expression: "spanKind", Type: "set(5)", Granularity: 4}, + {Name: "idx_stringTagMapKeys", Expression: "mapKeys(stringTagMap)", Type: "tokenbf_v1(1024, 2, 0)", Granularity: 1}, + {Name: "idx_stringTagMapValues", Expression: "mapValues(stringTagMap)", Type: "ngrambf_v1(4, 5000, 2, 0)", Granularity: 1}, + {Name: "idx_numberTagMapKeys", Expression: "mapKeys(numberTagMap)", Type: "tokenbf_v1(1024, 2, 0)", Granularity: 1}, + {Name: "idx_numberTagMapValues", Expression: "mapValues(numberTagMap)", Type: "bloom_filter(0.01)", Granularity: 1}, + {Name: "idx_boolTagMapKeys", Expression: "mapKeys(boolTagMap)", Type: "tokenbf_v1(1024, 2, 0)", Granularity: 1}, + {Name: "idx_resourceTagMapKeys", Expression: "mapKeys(resourceTagsMap)", Type: "tokenbf_v1(1024, 2, 0)", Granularity: 1}, + {Name: "idx_resourceTagMapValues", Expression: "mapValues(resourceTagsMap)", Type: "ngrambf_v1(4, 5000, 2, 0)", Granularity: 1}, + }, + Projections: []Projection{ + {Name: "timestampSort", Query: "SELECT * ORDER BY timestamp"}, + }, + Engine: MergeTree{ + PartitionBy: "toDate(timestamp)", + PrimaryKey: "(serviceName, hasError, toStartOfHour(timestamp), name)", + OrderBy: "(serviceName, hasError, toStartOfHour(timestamp), name, timestamp)", + TTL: "toDateTime(timestamp) + toIntervalSecond(604800)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "signoz_index_v2", + }, + }, + }, + { + MigrationID: 2, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "distributed_signoz_index_v2", + Columns: []Column{ + {Name: "timestamp", Type: DateTime64ColumnType{Precision: 9}, Codec: "DoubleDelta, LZ4"}, + {Name: "traceID", Type: FixedStringColumnType{Length: 32}, Codec: "ZSTD(1)"}, + {Name: "spanID", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "parentSpanID", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "serviceName", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "kind", Type: ColumnTypeInt8, Codec: "T64, ZSTD(1)"}, + {Name: "durationNano", Type: ColumnTypeUInt64, Codec: "T64, ZSTD(1)"}, + {Name: "statusCode", Type: ColumnTypeInt16, Codec: "T64, ZSTD(1)"}, + {Name: "externalHttpMethod", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "externalHttpUrl", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "dbSystem", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "dbName", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "dbOperation", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "peerService", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "events", Type: ArrayColumnType{ColumnTypeString}, Codec: "ZSTD(2)"}, + {Name: "httpMethod", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "httpUrl", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "httpRoute", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "httpHost", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "msgSystem", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "msgOperation", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "hasError", Type: ColumnTypeBool, Codec: "T64, ZSTD(1)"}, + {Name: "rpcSystem", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "rpcService", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "rpcMethod", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "responseStatusCode", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "stringTagMap", Type: MapColumnType{ColumnTypeString, ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "numberTagMap", Type: MapColumnType{ColumnTypeString, ColumnTypeFloat64}, Codec: "ZSTD(1)"}, + {Name: "boolTagMap", Type: MapColumnType{ColumnTypeString, ColumnTypeBool}, Codec: "ZSTD(1)"}, + {Name: "resourceTagsMap", Type: MapColumnType{LowCardinalityColumnType{ColumnTypeString}, ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "isRemote", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "statusMessage", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "statusCodeString", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "spanKind", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_traces", + Table: "signoz_index_v2", + ShardingKey: "cityHash64(traceID)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "distributed_signoz_index_v2", + }, + }, + }, + { + MigrationID: 3, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "durationSort", + Columns: []Column{ + {Name: "timestamp", Type: DateTime64ColumnType{Precision: 9}, Codec: "DoubleDelta, LZ4"}, + {Name: "traceID", Type: FixedStringColumnType{Length: 32}, Codec: "ZSTD(1)"}, + {Name: "spanID", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "parentSpanID", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "serviceName", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "kind", Type: ColumnTypeInt8, Codec: "T64, ZSTD(1)"}, + {Name: "durationNano", Type: ColumnTypeUInt64, Codec: "T64, ZSTD(1)"}, + {Name: "statusCode", Type: ColumnTypeInt16, Codec: "T64, ZSTD(1)"}, + {Name: "httpMethod", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "httpUrl", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "httpRoute", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "httpHost", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "hasError", Type: ColumnTypeBool, Codec: "T64, ZSTD(1)"}, + {Name: "rpcSystem", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "rpcService", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "rpcMethod", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "responseStatusCode", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "stringTagMap", Type: MapColumnType{ColumnTypeString, ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "numberTagMap", Type: MapColumnType{ColumnTypeString, ColumnTypeFloat64}, Codec: "ZSTD(1)"}, + {Name: "boolTagMap", Type: MapColumnType{ColumnTypeString, ColumnTypeBool}, Codec: "ZSTD(1)"}, + {Name: "isRemote", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "statusMessage", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "statusCodeString", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "spanKind", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + }, + Indexes: []Index{ + {Name: "idx_service", Expression: "serviceName", Type: "bloom_filter", Granularity: 4}, + {Name: "idx_name", Expression: "name", Type: "bloom_filter", Granularity: 4}, + {Name: "idx_kind", Expression: "kind", Type: "minmax", Granularity: 4}, + {Name: "idx_duration", Expression: "durationNano", Type: "minmax", Granularity: 1}, + {Name: "idx_hasError", Expression: "hasError", Type: "set(2)", Granularity: 1}, + {Name: "idx_httpRoute", Expression: "httpRoute", Type: "bloom_filter", Granularity: 4}, + {Name: "idx_httpUrl", Expression: "httpUrl", Type: "bloom_filter", Granularity: 4}, + {Name: "idx_httpHost", Expression: "httpHost", Type: "bloom_filter", Granularity: 4}, + {Name: "idx_httpMethod", Expression: "httpMethod", Type: "bloom_filter", Granularity: 4}, + {Name: "idx_timestamp", Expression: "timestamp", Type: "minmax", Granularity: 1}, + {Name: "idx_rpcMethod", Expression: "rpcMethod", Type: "bloom_filter", Granularity: 4}, + {Name: "idx_responseStatusCode", Expression: "responseStatusCode", Type: "set(0)", Granularity: 1}, + }, + Engine: MergeTree{ + PartitionBy: "toDate(timestamp)", + OrderBy: "(durationNano, timestamp)", + TTL: "toDateTime(timestamp) + toIntervalSecond(604800)", + Settings: TableSettings{ + { + Name: "index_granularity", + Value: "8192", + }, + { + Name: "ttl_only_drop_parts", + Value: "1", + }, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "durationSort", + }, + }, + }, + { + MigrationID: 4, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_traces", + ViewName: "durationSortMV", + DestTable: "durationSort", + Columns: []Column{ + {Name: "timestamp", Type: DateTime64ColumnType{Precision: 9}, Codec: "DoubleDelta, LZ4"}, + {Name: "traceID", Type: FixedStringColumnType{Length: 32}, Codec: "ZSTD(1)"}, + {Name: "spanID", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "parentSpanID", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "serviceName", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "kind", Type: ColumnTypeInt8, Codec: "T64, ZSTD(1)"}, + {Name: "durationNano", Type: ColumnTypeUInt64, Codec: "T64, ZSTD(1)"}, + {Name: "statusCode", Type: ColumnTypeInt16, Codec: "T64, ZSTD(1)"}, + {Name: "httpMethod", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "httpUrl", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "httpRoute", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "httpHost", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "hasError", Type: ColumnTypeBool, Codec: "T64, ZSTD(1)"}, + {Name: "rpcSystem", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "rpcService", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "rpcMethod", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "responseStatusCode", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "stringTagMap", Type: MapColumnType{ColumnTypeString, ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "numberTagMap", Type: MapColumnType{ColumnTypeString, ColumnTypeFloat64}, Codec: "ZSTD(1)"}, + {Name: "boolTagMap", Type: MapColumnType{ColumnTypeString, ColumnTypeBool}, Codec: "ZSTD(1)"}, + {Name: "isRemote", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "statusMessage", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "statusCodeString", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "spanKind", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + }, + Query: `SELECT + timestamp, + traceID, + spanID, + parentSpanID, + serviceName, + name, + kind, + durationNano, + statusCode, + httpMethod, + httpUrl, + httpRoute, + httpHost, + hasError, + rpcSystem, + rpcService, + rpcMethod, + responseStatusCode, + stringTagMap, + numberTagMap, + boolTagMap, + isRemote, + statusMessage, + statusCodeString, + spanKind +FROM signoz_traces.signoz_index_v2 +ORDER BY + durationNano ASC, + timestamp ASC`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "durationSortMV", + }, + }, + }, + { + MigrationID: 5, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "distributed_durationSort", + Columns: []Column{ + {Name: "timestamp", Type: DateTime64ColumnType{Precision: 9}, Codec: "DoubleDelta, LZ4"}, + {Name: "traceID", Type: FixedStringColumnType{Length: 32}, Codec: "ZSTD(1)"}, + {Name: "spanID", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "parentSpanID", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "serviceName", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "kind", Type: ColumnTypeInt8, Codec: "T64, ZSTD(1)"}, + {Name: "durationNano", Type: ColumnTypeUInt64, Codec: "T64, ZSTD(1)"}, + {Name: "statusCode", Type: ColumnTypeInt16, Codec: "T64, ZSTD(1)"}, + {Name: "httpMethod", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "httpUrl", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "httpRoute", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "httpHost", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "hasError", Type: ColumnTypeBool, Codec: "T64, ZSTD(1)"}, + {Name: "rpcSystem", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "rpcService", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "rpcMethod", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "responseStatusCode", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "stringTagMap", Type: MapColumnType{ColumnTypeString, ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "numberTagMap", Type: MapColumnType{ColumnTypeString, ColumnTypeFloat64}, Codec: "ZSTD(1)"}, + {Name: "boolTagMap", Type: MapColumnType{ColumnTypeString, ColumnTypeBool}, Codec: "ZSTD(1)"}, + {Name: "isRemote", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "statusMessage", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "statusCodeString", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "spanKind", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_traces", + Table: "durationSort", + ShardingKey: "cityHash64(traceID)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "distributed_durationSort", + }, + }, + }, + { + MigrationID: 6, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "signoz_error_index_v2", + Columns: []Column{ + {Name: "timestamp", Type: DateTime64ColumnType{Precision: 9}, Codec: "DoubleDelta, LZ4"}, + {Name: "errorID", Type: FixedStringColumnType{Length: 32}, Codec: "ZSTD(1)"}, + {Name: "groupID", Type: FixedStringColumnType{Length: 32}, Codec: "ZSTD(1)"}, + {Name: "traceID", Type: FixedStringColumnType{Length: 32}, Codec: "ZSTD(1)"}, + {Name: "spanID", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "serviceName", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "exceptionType", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "exceptionMessage", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "exceptionStacktrace", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "exceptionEscaped", Type: ColumnTypeBool, Codec: "T64, ZSTD(1)"}, + {Name: "resourceTagsMap", Type: MapColumnType{LowCardinalityColumnType{ColumnTypeString}, ColumnTypeString}, Codec: "ZSTD(1)"}, + }, + Indexes: []Index{ + {Name: "idx_error_id", Expression: "errorID", Type: "bloom_filter", Granularity: 4}, + {Name: "idx_resourceTagsMapKeys", Expression: "mapKeys(resourceTagsMap)", Type: "bloom_filter(0.01)", Granularity: 64}, + {Name: "idx_resourceTagsMapValues", Expression: "mapValues(resourceTagsMap)", Type: "bloom_filter(0.01)", Granularity: 64}, + }, + Engine: MergeTree{ + PartitionBy: "toDate(timestamp)", + OrderBy: "(timestamp, groupID)", + TTL: "toDateTime(timestamp) + toIntervalSecond(604800)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "signoz_error_index_v2", + }, + }, + }, + { + MigrationID: 7, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "distributed_signoz_error_index_v2", + Columns: []Column{ + {Name: "timestamp", Type: DateTime64ColumnType{Precision: 9}, Codec: "DoubleDelta, LZ4"}, + {Name: "errorID", Type: FixedStringColumnType{Length: 32}, Codec: "ZSTD(1)"}, + {Name: "groupID", Type: FixedStringColumnType{Length: 32}, Codec: "ZSTD(1)"}, + {Name: "traceID", Type: FixedStringColumnType{Length: 32}, Codec: "ZSTD(1)"}, + {Name: "spanID", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "serviceName", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "exceptionType", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "exceptionMessage", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "exceptionStacktrace", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "exceptionEscaped", Type: ColumnTypeBool, Codec: "T64, ZSTD(1)"}, + {Name: "resourceTagsMap", Type: MapColumnType{LowCardinalityColumnType{ColumnTypeString}, ColumnTypeString}, Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_traces", + Table: "signoz_error_index_v2", + ShardingKey: "cityHash64(groupID)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "distributed_signoz_error_index_v2", + }, + }, + }, + { + MigrationID: 8, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "signoz_spans", + Columns: []Column{ + {Name: "timestamp", Type: DateTime64ColumnType{Precision: 9}, Codec: "DoubleDelta, LZ4"}, + {Name: "traceID", Type: FixedStringColumnType{Length: 32}, Codec: "ZSTD(1)"}, + {Name: "model", Type: ColumnTypeString, Codec: "ZSTD(9)"}, + }, + Engine: MergeTree{ + PartitionBy: "toDate(timestamp)", + OrderBy: "(traceID)", + TTL: "toDateTime(timestamp) + toIntervalSecond(604800)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "1024"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "signoz_spans", + }, + }, + }, + { + MigrationID: 9, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "distributed_signoz_spans", + Columns: []Column{ + {Name: "timestamp", Type: DateTime64ColumnType{Precision: 9}, Codec: "DoubleDelta, LZ4"}, + {Name: "traceID", Type: FixedStringColumnType{Length: 32}, Codec: "ZSTD(1)"}, + {Name: "model", Type: ColumnTypeString, Codec: "ZSTD(9)"}, + }, + Engine: Distributed{ + Database: "signoz_traces", + Table: "signoz_spans", + ShardingKey: "cityHash64(traceID)", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "distributed_signoz_spans", + }, + }, + }, + { + MigrationID: 10, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "dependency_graph_minutes_v2", + Columns: []Column{ + {Name: "src", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "dest", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "duration_quantiles_state", Type: AggregateFunction{FunctionName: "quantiles(0.5, 0.75, 0.9, 0.95, 0.99)", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "Default"}, + {Name: "error_count", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "T64, ZSTD(1)"}, + {Name: "total_count", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "T64, ZSTD(1)"}, + {Name: "timestamp", Type: DateTimeColumnType{}, Codec: "DoubleDelta, LZ4"}, + {Name: "deployment_environment", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "k8s_cluster_name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "k8s_namespace_name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + }, + Engine: AggregatingMergeTree{MergeTree{ + PartitionBy: "toDate(timestamp)", + OrderBy: "(timestamp, src, dest, deployment_environment, k8s_cluster_name, k8s_namespace_name)", + TTL: "toDateTime(timestamp) + toIntervalSecond(604800)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + }, + }}, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "dependency_graph_minutes_v2", + }, + }, + }, + { + MigrationID: 11, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_traces", + ViewName: "dependency_graph_minutes_db_calls_mv_v2", + DestTable: "dependency_graph_minutes_v2", + Columns: []Column{ + {Name: "src", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "dest", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "duration_quantiles_state", Type: AggregateFunction{FunctionName: "quantiles(0.5, 0.75, 0.9, 0.95, 0.99)", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "Default"}, + {Name: "error_count", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "T64, ZSTD(1)"}, + {Name: "total_count", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "T64, ZSTD(1)"}, + {Name: "timestamp", Type: DateTimeColumnType{}, Codec: "DoubleDelta, LZ4"}, + {Name: "deployment_environment", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "k8s_cluster_name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "k8s_namespace_name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + }, + Query: `SELECT + serviceName AS src, + dbSystem AS dest, + quantilesState(0.5, 0.75, 0.9, 0.95, 0.99)(toFloat64(durationNano)) AS duration_quantiles_state, + countIf(statusCode = 2) AS error_count, + count(*) AS total_count, + toStartOfMinute(timestamp) AS timestamp, + resourceTagsMap['deployment.environment'] AS deployment_environment, + resourceTagsMap['k8s.cluster.name'] AS k8s_cluster_name, + resourceTagsMap['k8s.namespace.name'] AS k8s_namespace_name +FROM signoz_traces.signoz_index_v2 +WHERE (dest != '') AND (kind != 2) +GROUP BY + timestamp, + src, + dest, + deployment_environment, + k8s_cluster_name, + k8s_namespace_name`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "dependency_graph_minutes_db_calls_mv_v2", + }, + }, + }, + { + MigrationID: 12, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_traces", + ViewName: "dependency_graph_minutes_messaging_calls_mv_v2", + DestTable: "dependency_graph_minutes_v2", + Columns: []Column{ + {Name: "src", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "dest", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "duration_quantiles_state", Type: AggregateFunction{FunctionName: "quantiles(0.5, 0.75, 0.9, 0.95, 0.99)", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "Default"}, + {Name: "error_count", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "T64, ZSTD(1)"}, + {Name: "total_count", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "T64, ZSTD(1)"}, + {Name: "timestamp", Type: DateTimeColumnType{}, Codec: "DoubleDelta, LZ4"}, + {Name: "deployment_environment", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "k8s_cluster_name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "k8s_namespace_name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + }, + Query: `SELECT + serviceName AS src, + msgSystem AS dest, + quantilesState(0.5, 0.75, 0.9, 0.95, 0.99)(toFloat64(durationNano)) AS duration_quantiles_state, + countIf(statusCode = 2) AS error_count, + count(*) AS total_count, + toStartOfMinute(timestamp) AS timestamp, + resourceTagsMap['deployment.environment'] AS deployment_environment, + resourceTagsMap['k8s.cluster.name'] AS k8s_cluster_name, + resourceTagsMap['k8s.namespace.name'] AS k8s_namespace_name +FROM signoz_traces.signoz_index_v2 +WHERE (dest != '') AND (kind != 2) +GROUP BY + timestamp, + src, + dest, + deployment_environment, + k8s_cluster_name, + k8s_namespace_name`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "dependency_graph_minutes_messaging_calls_mv_v2", + }, + }, + }, + { + MigrationID: 13, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_traces", + ViewName: "dependency_graph_minutes_service_calls_mv_v2", + DestTable: "dependency_graph_minutes_v2", + Columns: []Column{ + {Name: "src", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "dest", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "duration_quantiles_state", Type: AggregateFunction{FunctionName: "quantiles(0.5, 0.75, 0.9, 0.95, 0.99)", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "Default"}, + {Name: "error_count", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "T64, ZSTD(1)"}, + {Name: "total_count", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "T64, ZSTD(1)"}, + {Name: "timestamp", Type: DateTimeColumnType{}, Codec: "DoubleDelta, LZ4"}, + {Name: "deployment_environment", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "k8s_cluster_name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "k8s_namespace_name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + }, + Query: `SELECT + A.serviceName AS src, + B.serviceName AS dest, + quantilesState(0.5, 0.75, 0.9, 0.95, 0.99)(toFloat64(B.durationNano)) AS duration_quantiles_state, + countIf(B.statusCode = 2) AS error_count, + count(*) AS total_count, + toStartOfMinute(B.timestamp) AS timestamp, + B.resourceTagsMap['deployment.environment'] AS deployment_environment, + B.resourceTagsMap['k8s.cluster.name'] AS k8s_cluster_name, + B.resourceTagsMap['k8s.namespace.name'] AS k8s_namespace_name +FROM signoz_traces.signoz_index_v2 AS A, signoz_traces.signoz_index_v2 AS B +WHERE (A.serviceName != B.serviceName) AND (A.spanID = B.parentSpanID) +GROUP BY + timestamp, + src, + dest, + deployment_environment, + k8s_cluster_name, + k8s_namespace_name`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "dependency_graph_minutes_service_calls_mv_v2", + }, + }, + }, + { + MigrationID: 14, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "distributed_dependency_graph_minutes_v2", + Columns: []Column{ + {Name: "src", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "dest", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "duration_quantiles_state", Type: AggregateFunction{FunctionName: "quantiles(0.5, 0.75, 0.9, 0.95, 0.99)", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "Default"}, + {Name: "error_count", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "T64, ZSTD(1)"}, + {Name: "total_count", Type: SimpleAggregateFunction{FunctionName: "sum", Arguments: []ColumnType{ColumnTypeUInt64}}, Codec: "T64, ZSTD(1)"}, + {Name: "timestamp", Type: DateTimeColumnType{}, Codec: "DoubleDelta, LZ4"}, + {Name: "deployment_environment", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "k8s_cluster_name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "k8s_namespace_name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_traces", + Table: "dependency_graph_minutes_v2", + ShardingKey: "cityHash64(rand())", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "distributed_dependency_graph_minutes_v2", + }, + }, + }, + { + MigrationID: 15, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "usage_explorer", + Columns: []Column{ + {Name: "timestamp", Type: DateTime64ColumnType{Precision: 9}, Codec: "DoubleDelta, LZ4"}, + {Name: "service_name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "count", Type: ColumnTypeUInt64, Codec: "T64, ZSTD(1)"}, + }, + Engine: SummingMergeTree{MergeTree{ + PartitionBy: "toDate(timestamp)", + OrderBy: "(timestamp, service_name)", + TTL: "toDateTime(timestamp) + toIntervalSecond(604800)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + }, + }}, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "usage_explorer", + }, + }, + }, + { + MigrationID: 16, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_traces", + ViewName: "usage_explorer_mv", + DestTable: "usage_explorer", + Columns: []Column{ + {Name: "timestamp", Type: DateTime64ColumnType{Precision: 9}, Codec: "DoubleDelta, LZ4"}, + {Name: "service_name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "count", Type: ColumnTypeUInt64, Codec: "T64, ZSTD(1)"}, + }, + Query: `SELECT + toStartOfHour(timestamp) AS timestamp, + serviceName AS service_name, + count() AS count +FROM signoz_traces.signoz_index_v2 +GROUP BY + timestamp, + serviceName`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "usage_explorer_mv", + }, + }, + }, + { + MigrationID: 17, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "distributed_usage_explorer", + Columns: []Column{ + {Name: "timestamp", Type: DateTime64ColumnType{Precision: 9}, Codec: "DoubleDelta, LZ4"}, + {Name: "service_name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "count", Type: ColumnTypeUInt64, Codec: "T64, ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_traces", + Table: "usage_explorer", + ShardingKey: "cityHash64(rand())", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "distributed_usage_explorer", + }, + }, + }, + { + MigrationID: 18, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "top_level_operations", + Columns: []Column{ + {Name: "name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "serviceName", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "time", Type: DateTimeColumnType{}, Default: "now()", Codec: "ZSTD(1)"}, + }, + Engine: ReplacingMergeTree{ + MergeTree: MergeTree{ + OrderBy: "(serviceName, name)", + TTL: "time + toIntervalMonth(1)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + }, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "top_level_operations", + }, + }, + }, + { + MigrationID: 19, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "distributed_top_level_operations", + Columns: []Column{ + {Name: "name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "serviceName", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "time", Type: DateTimeColumnType{}, Default: "now()", Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_traces", + Table: "top_level_operations", + ShardingKey: "cityHash64(rand())", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "distributed_top_level_operations", + }, + }, + }, + { + MigrationID: 20, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_traces", + ViewName: "root_operations", + DestTable: "top_level_operations", + Columns: []Column{ + {Name: "name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "serviceName", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + }, + Query: `SELECT DISTINCT + name, + serviceName +FROM signoz_traces.signoz_index_v2 +WHERE parentSpanID = ''`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "root_operations", + }, + }, + }, + { + MigrationID: 21, + UpItems: []Operation{ + CreateMaterializedViewOperation{ + Database: "signoz_traces", + ViewName: "sub_root_operations", + DestTable: "top_level_operations", + Columns: []Column{ + {Name: "name", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "serviceName", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + }, + Query: `SELECT DISTINCT + name, + serviceName +FROM signoz_traces.signoz_index_v2 AS A, signoz_traces.signoz_index_v2 AS B +WHERE (A.serviceName != B.serviceName) AND (A.parentSpanID = B.spanID)`, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "sub_root_operations", + }, + }, + }, + { + MigrationID: 22, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "span_attributes", + Columns: []Column{ + {Name: "timestamp", Type: DateTimeColumnType{}, Codec: "DoubleDelta, ZSTD(1)"}, + {Name: "tagKey", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "tagType", Type: EnumerationColumnType{Values: []string{"'tag' = 1", "'resource' = 2"}, Size: 8}, Codec: "ZSTD(1)"}, + {Name: "dataType", Type: EnumerationColumnType{Values: []string{"'string' = 1", "'bool' = 2", "'float64' = 3"}, Size: 8}, Codec: "ZSTD(1)"}, + {Name: "stringTagValue", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "float64TagValue", Type: NullableColumnType{ColumnTypeFloat64}, Codec: "ZSTD(1)"}, + {Name: "isColumn", Type: ColumnTypeBool, Codec: "ZSTD(1)"}, + }, + Engine: ReplacingMergeTree{ + MergeTree: MergeTree{ + OrderBy: "(tagKey, tagType, dataType, stringTagValue, float64TagValue, isColumn)", + TTL: "toDateTime(timestamp) + toIntervalSecond(172800)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + {Name: "ttl_only_drop_parts", Value: "1"}, + {Name: "allow_nullable_key", Value: "1"}, + }, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "span_attributes", + }, + }, + }, + { + MigrationID: 23, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "distributed_span_attributes", + Columns: []Column{ + {Name: "timestamp", Type: DateTimeColumnType{}, Codec: "DoubleDelta, ZSTD(1)"}, + {Name: "tagKey", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "tagType", Type: EnumerationColumnType{Values: []string{"'tag' = 1", "'resource' = 2"}, Size: 8}, Codec: "ZSTD(1)"}, + {Name: "dataType", Type: EnumerationColumnType{Values: []string{"'string' = 1", "'bool' = 2", "'float64' = 3"}, Size: 8}, Codec: "ZSTD(1)"}, + {Name: "stringTagValue", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "float64TagValue", Type: NullableColumnType{ColumnTypeFloat64}, Codec: "ZSTD(1)"}, + {Name: "isColumn", Type: ColumnTypeBool, Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_traces", + Table: "span_attributes", + ShardingKey: "cityHash64(rand())", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "distributed_span_attributes", + }, + }, + }, + { + MigrationID: 24, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "span_attributes_keys", + Columns: []Column{ + {Name: "tagKey", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "tagType", Type: EnumerationColumnType{Values: []string{"'tag' = 1", "'resource' = 2"}, Size: 8}, Codec: "ZSTD(1)"}, + {Name: "dataType", Type: EnumerationColumnType{Values: []string{"'string' = 1", "'bool' = 2", "'float64' = 3"}, Size: 8}, Codec: "ZSTD(1)"}, + {Name: "isColumn", Type: ColumnTypeBool, Codec: "ZSTD(1)"}, + }, + Engine: ReplacingMergeTree{ + MergeTree: MergeTree{ + OrderBy: "(tagKey, tagType, dataType, isColumn)", + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "span_attributes_keys", + }, + }, + }, + { + MigrationID: 25, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "distributed_span_attributes_keys", + Columns: []Column{ + {Name: "tagKey", Type: LowCardinalityColumnType{ColumnTypeString}, Codec: "ZSTD(1)"}, + {Name: "tagType", Type: EnumerationColumnType{Values: []string{"'tag' = 1", "'resource' = 2"}, Size: 8}, Codec: "ZSTD(1)"}, + {Name: "dataType", Type: EnumerationColumnType{Values: []string{"'string' = 1", "'bool' = 2", "'float64' = 3"}, Size: 8}, Codec: "ZSTD(1)"}, + {Name: "isColumn", Type: ColumnTypeBool, Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_traces", + Table: "span_attributes_keys", + ShardingKey: "cityHash64(rand())", + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "distributed_span_attributes_keys", + }, + }, + }, + { + MigrationID: 26, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "usage", + Columns: []Column{ + {Name: "tenant", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "collector_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "exporter_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "timestamp", Type: DateTimeColumnType{}, Codec: "ZSTD(1)"}, + {Name: "data", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + }, + Engine: MergeTree{ + OrderBy: "(tenant, collector_id, exporter_id, timestamp)", + TTL: "timestamp + toIntervalDay(3)", + Settings: TableSettings{ + {Name: "index_granularity", Value: "8192"}, + }, + }, + }, + }, + DownItems: []Operation{ + DropTableOperation{ + Database: "signoz_traces", + Table: "usage", + }, + }, + }, + { + MigrationID: 27, + UpItems: []Operation{ + CreateTableOperation{ + Database: "signoz_traces", + Table: "distributed_usage", + Columns: []Column{ + {Name: "tenant", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "collector_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "exporter_id", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + {Name: "timestamp", Type: DateTimeColumnType{}, Codec: "ZSTD(1)"}, + {Name: "data", Type: ColumnTypeString, Codec: "ZSTD(1)"}, + }, + Engine: Distributed{ + Database: "signoz_traces", + Table: "usage", + ShardingKey: "cityHash64(rand())", + }, + }, + }, + }, + } +)