From e6bd49b32c82653108319001727104b20a6f4357 Mon Sep 17 00:00:00 2001 From: dhawal1248 Date: Tue, 10 Oct 2023 17:05:32 +0530 Subject: [PATCH] feat: separate migrations from codebase + templatize migrations --- cmd/signozcollectormigrator/migrate.go | 74 ++++++++ exporter/clickhouselogsexporter/exporter.go | 83 --------- .../migrations/000002_add_minmax_idx.down.sql | 1 - .../migrations/000002_add_minmax_idx.up.sql | 1 - .../000003_add_distributed_table.down.sql | 3 - .../000004_ttl_only_drop_parts.down.sql | 1 - .../000004_ttl_only_drop_parts.up.sql | 1 - .../migrations/000006_tag_attributes.down.sql | 2 - .../000007_default_indexes.down.sql | 5 - .../clickhousemetricsexporter/clickhouse.go | 97 ----------- .../clickhouse_factory.go | 164 ------------------ .../migrations/000001_init_db.down.sql | 1 - .../migrations/000002_init_schema.down.sql | 1 - .../migrations/000002_init_schema.up.sql | 1 - .../migrations/000004_update_table.down.sql | 1 - .../migrations/000004_update_table.up.sql | 1 - .../migrations/000005_add_hasError.down.sql | 1 - .../migrations/000005_add_hasError.up.sql | 1 - .../migrations/000006_add_tag_map.down.sql | 1 - .../migrations/000006_add_tag_map.up.sql | 1 - .../000007_init_signoz_index_v2.down.sql | 3 - .../000008_init_signoz_spans.down.sql | 1 - .../000010_add_gRPC_code_method.down.sql | 1 - .../000010_add_gRPC_code_method.up.sql | 1 - .../000011_init_durationMV.down.sql | 2 - .../000014_add_usage_explorer_mv.down.sql | 2 - .../000019_ttl_only_drop_parts.down.sql | 6 - .../000019_ttl_only_drop_parts.up.sql | 6 - .../000023_span_attributes_mv.down.sql | 2 - migrator/manager.go | 86 +++++++++ migrator/migrators/baseMigrator/migrator.go | 141 +++++++++++++++ .../logs}/migrations/000001_init_db.down.sql | 14 +- .../logs}/migrations/000001_init_db.up.sql | 15 +- .../migrations/000002_add_minmax_idx.down.sql | 1 + .../migrations/000002_add_minmax_idx.up.sql | 1 + .../000003_add_distributed_table.down.sql | 3 + .../000003_add_distributed_table.up.sql | 6 +- .../000004_ttl_only_drop_parts.down.sql | 1 + .../000004_ttl_only_drop_parts.up.sql | 1 + .../000005_attribute_rename.down.sql | 18 +- .../migrations/000005_attribute_rename.up.sql | 18 +- .../migrations/000006_tag_attributes.down.sql | 2 + .../migrations/000006_tag_attributes.up.sql | 4 +- .../000007_default_indexes.down.sql | 5 + .../migrations/000007_default_indexes.up.sql | 6 +- migrator/migrators/logs/migrator.go | 24 +++ .../migrations/000001_init_db.down.sql | 11 ++ .../metrics/migrations/000001_init_db.up.sql | 55 ++++++ migrator/migrators/metrics/migrator.go | 40 +++++ migrator/migrators/migrators.go | 18 ++ .../traces/migrations/000001_init_db.down.sql | 1 + .../traces}/migrations/000001_init_db.up.sql | 2 +- .../migrations/000002_init_schema.down.sql | 1 + .../migrations/000002_init_schema.up.sql | 1 + .../migrations/000003_init_schema.down.sql | 2 +- .../migrations/000003_init_schema.up.sql | 2 +- .../migrations/000004_update_table.down.sql | 1 + .../migrations/000004_update_table.up.sql | 1 + .../migrations/000005_add_hasError.down.sql | 1 + .../migrations/000005_add_hasError.up.sql | 1 + .../migrations/000006_add_tag_map.down.sql | 1 + .../migrations/000006_add_tag_map.up.sql | 1 + .../000007_init_signoz_index_v2.down.sql | 3 + .../000007_init_signoz_index_v2.up.sql | 2 +- .../000008_init_signoz_spans.down.sql | 1 + .../000008_init_signoz_spans.up.sql | 2 +- .../000009_init_signoz_error_index.down.sql | 2 +- .../000009_init_signoz_error_index.up.sql | 2 +- .../000010_add_gRPC_code_method.down.sql | 1 + .../000010_add_gRPC_code_method.up.sql | 1 + .../000011_init_durationMV.down.sql | 2 + .../migrations/000011_init_durationMV.up.sql | 4 +- ...000012_init_signoz_error_index_v2.down.sql | 2 +- .../000012_init_signoz_error_index_v2.up.sql | 2 +- ...013_add_RPC_system_service_method.down.sql | 10 +- ...00013_add_RPC_system_service_method.up.sql | 12 +- .../000014_add_usage_explorer_mv.down.sql | 2 + .../000014_add_usage_explorer_mv.up.sql | 4 +- .../000015_top_level_operations_mv.down.sql | 6 +- .../000015_top_level_operations_mv.up.sql | 6 +- .../000016_dependency_graph_minutes.down.sql | 8 +- .../000016_dependency_graph_minutes.up.sql | 8 +- .../000017_add_distributed_table.down.sql | 14 +- .../000017_add_distributed_table.up.sql | 14 +- .../000018_add_new_tag_types.down.sql | 12 +- .../000018_add_new_tag_types.up.sql | 12 +- .../000019_ttl_only_drop_parts.down.sql | 6 + .../000019_ttl_only_drop_parts.up.sql | 6 + .../000020_resource_att_exceptions.down.sql | 4 +- .../000020_resource_att_exceptions.up.sql | 4 +- ...source_attributes_signoz_index_v2.down.sql | 4 +- ...resource_attributes_signoz_index_v2.up.sql | 4 +- ...00022_dependency_graph_minutes_v2.down.sql | 10 +- .../000022_dependency_graph_minutes_v2.up.sql | 10 +- .../000023_span_attributes_mv.down.sql | 2 + .../000023_span_attributes_mv.up.sql | 4 +- .../000024_span_attributes_keys.down.sql | 4 +- .../000024_span_attributes_keys.up.sql | 4 +- migrator/migrators/traces/migrator.go | 25 +++ 99 files changed, 649 insertions(+), 520 deletions(-) create mode 100644 cmd/signozcollectormigrator/migrate.go delete mode 100644 exporter/clickhouselogsexporter/migrations/000002_add_minmax_idx.down.sql delete mode 100644 exporter/clickhouselogsexporter/migrations/000002_add_minmax_idx.up.sql delete mode 100644 exporter/clickhouselogsexporter/migrations/000003_add_distributed_table.down.sql delete mode 100644 exporter/clickhouselogsexporter/migrations/000004_ttl_only_drop_parts.down.sql delete mode 100644 exporter/clickhouselogsexporter/migrations/000004_ttl_only_drop_parts.up.sql delete mode 100644 exporter/clickhouselogsexporter/migrations/000006_tag_attributes.down.sql delete mode 100644 exporter/clickhouselogsexporter/migrations/000007_default_indexes.down.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000001_init_db.down.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000002_init_schema.down.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000002_init_schema.up.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000004_update_table.down.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000004_update_table.up.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000005_add_hasError.down.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000005_add_hasError.up.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000006_add_tag_map.down.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000006_add_tag_map.up.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000007_init_signoz_index_v2.down.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000008_init_signoz_spans.down.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000010_add_gRPC_code_method.down.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000010_add_gRPC_code_method.up.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000011_init_durationMV.down.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000014_add_usage_explorer_mv.down.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000019_ttl_only_drop_parts.down.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000019_ttl_only_drop_parts.up.sql delete mode 100644 exporter/clickhousetracesexporter/migrations/000023_span_attributes_mv.down.sql create mode 100644 migrator/manager.go create mode 100644 migrator/migrators/baseMigrator/migrator.go rename {exporter/clickhouselogsexporter => migrator/migrators/logs}/migrations/000001_init_db.down.sql (51%) rename {exporter/clickhouselogsexporter => migrator/migrators/logs}/migrations/000001_init_db.up.sql (82%) create mode 100644 migrator/migrators/logs/migrations/000002_add_minmax_idx.down.sql create mode 100644 migrator/migrators/logs/migrations/000002_add_minmax_idx.up.sql create mode 100644 migrator/migrators/logs/migrations/000003_add_distributed_table.down.sql rename {exporter/clickhouselogsexporter => migrator/migrators/logs}/migrations/000003_add_distributed_table.up.sql (70%) create mode 100644 migrator/migrators/logs/migrations/000004_ttl_only_drop_parts.down.sql create mode 100644 migrator/migrators/logs/migrations/000004_ttl_only_drop_parts.up.sql rename {exporter/clickhouselogsexporter => migrator/migrators/logs}/migrations/000005_attribute_rename.down.sql (57%) rename {exporter/clickhouselogsexporter => migrator/migrators/logs}/migrations/000005_attribute_rename.up.sql (57%) create mode 100644 migrator/migrators/logs/migrations/000006_tag_attributes.down.sql rename {exporter/clickhouselogsexporter => migrator/migrators/logs}/migrations/000006_tag_attributes.up.sql (90%) create mode 100644 migrator/migrators/logs/migrations/000007_default_indexes.down.sql rename {exporter/clickhouselogsexporter => migrator/migrators/logs}/migrations/000007_default_indexes.up.sql (52%) create mode 100644 migrator/migrators/logs/migrator.go create mode 100644 migrator/migrators/metrics/migrations/000001_init_db.down.sql create mode 100644 migrator/migrators/metrics/migrations/000001_init_db.up.sql create mode 100644 migrator/migrators/metrics/migrator.go create mode 100644 migrator/migrators/migrators.go create mode 100644 migrator/migrators/traces/migrations/000001_init_db.down.sql rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000001_init_db.up.sql (98%) create mode 100644 migrator/migrators/traces/migrations/000002_init_schema.down.sql create mode 100644 migrator/migrators/traces/migrations/000002_init_schema.up.sql rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000003_init_schema.down.sql (75%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000003_init_schema.up.sql (96%) create mode 100644 migrator/migrators/traces/migrations/000004_update_table.down.sql create mode 100644 migrator/migrators/traces/migrations/000004_update_table.up.sql create mode 100644 migrator/migrators/traces/migrations/000005_add_hasError.down.sql create mode 100644 migrator/migrators/traces/migrations/000005_add_hasError.up.sql create mode 100644 migrator/migrators/traces/migrations/000006_add_tag_map.down.sql create mode 100644 migrator/migrators/traces/migrations/000006_add_tag_map.up.sql create mode 100644 migrator/migrators/traces/migrations/000007_init_signoz_index_v2.down.sql rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000007_init_signoz_index_v2.up.sql (98%) create mode 100644 migrator/migrators/traces/migrations/000008_init_signoz_spans.down.sql rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000008_init_signoz_spans.up.sql (93%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000009_init_signoz_error_index.down.sql (75%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000009_init_signoz_error_index.up.sql (96%) create mode 100644 migrator/migrators/traces/migrations/000010_add_gRPC_code_method.down.sql create mode 100644 migrator/migrators/traces/migrations/000010_add_gRPC_code_method.up.sql create mode 100644 migrator/migrators/traces/migrations/000011_init_durationMV.down.sql rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000011_init_durationMV.up.sql (97%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000012_init_signoz_error_index_v2.down.sql (72%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000012_init_signoz_error_index_v2.up.sql (95%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000013_add_RPC_system_service_method.down.sql (68%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000013_add_RPC_system_service_method.up.sql (78%) create mode 100644 migrator/migrators/traces/migrations/000014_add_usage_explorer_mv.down.sql rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000014_add_usage_explorer_mv.up.sql (90%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000015_top_level_operations_mv.down.sql (50%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000015_top_level_operations_mv.up.sql (86%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000016_dependency_graph_minutes.down.sql (61%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000016_dependency_graph_minutes.up.sql (90%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000017_add_distributed_table.down.sql (66%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000017_add_distributed_table.up.sql (68%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000018_add_new_tag_types.down.sql (79%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000018_add_new_tag_types.up.sql (79%) create mode 100644 migrator/migrators/traces/migrations/000019_ttl_only_drop_parts.down.sql create mode 100644 migrator/migrators/traces/migrations/000019_ttl_only_drop_parts.up.sql rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000020_resource_att_exceptions.down.sql (58%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000020_resource_att_exceptions.up.sql (82%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000021_resource_attributes_signoz_index_v2.down.sql (61%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000021_resource_attributes_signoz_index_v2.up.sql (84%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000022_dependency_graph_minutes_v2.down.sql (60%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000022_dependency_graph_minutes_v2.up.sql (91%) create mode 100644 migrator/migrators/traces/migrations/000023_span_attributes_mv.down.sql rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000023_span_attributes_mv.up.sql (89%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000024_span_attributes_keys.down.sql (68%) rename {exporter/clickhousetracesexporter => migrator/migrators/traces}/migrations/000024_span_attributes_keys.up.sql (81%) create mode 100644 migrator/migrators/traces/migrator.go diff --git a/cmd/signozcollectormigrator/migrate.go b/cmd/signozcollectormigrator/migrate.go new file mode 100644 index 00000000..93a2d841 --- /dev/null +++ b/cmd/signozcollectormigrator/migrate.go @@ -0,0 +1,74 @@ +package main + +import ( + "context" + "fmt" + "log" + "os" + + "github.com/SigNoz/signoz-otel-collector/migrator" + "github.com/spf13/pflag" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +func init() { + // init zap logger + config := zap.NewProductionConfig() + config.EncoderConfig.EncodeLevel = zapcore.LowercaseLevelEncoder + config.EncoderConfig.TimeKey = "timestamp" + config.EncoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder + logger, err := config.Build() + if err != nil { + log.Fatalf("Failed to initialize zap logger %v", err) + } + // replace global logger + zap.ReplaceGlobals(logger) +} + +func main() { + logger := zap.L().With(zap.String("component", "migrate cli")) + f := pflag.NewFlagSet("Collector Migrator CLI Options", pflag.ExitOnError) + + f.Usage = func() { + fmt.Println(f.FlagUsages()) + os.Exit(0) + } + + f.String("dsn", "", "Clickhouse DSN") + f.String("cluster-name", "", "Cluster name to use while running migrations") + f.Bool("multi-node-cluster", false, "True if the dsn points to a multi node clickhouse cluster, false otherwise. Defaults to false.") + + err := f.Parse(os.Args[1:]) + if err != nil { + logger.Fatal("Failed to parse args", zap.Error(err)) + } + + dsn, err := f.GetString("dsn") + if err != nil { + logger.Fatal("Failed to get dsn from args", zap.Error(err)) + } + + clusterName, err := f.GetString("cluster-name") + if err != nil { + logger.Fatal("Failed to get cluster name from args", zap.Error(err)) + } + + multiNodeCluster, err := f.GetBool("multi-node-cluster") + if err != nil { + logger.Fatal("Failed to get multi node cluster flag from args", zap.Error(err)) + } + + if dsn == "" || clusterName == "" { + logger.Fatal("dsn and clusterName are required fields") + } + + migrationManager, err := migrator.NewMigrationManager(dsn, clusterName, multiNodeCluster) + if err != nil { + logger.Fatal("Failed to create migration manager", zap.Error(err)) + } + err = migrationManager.Migrate(context.Background()) + if err != nil { + logger.Fatal("Failed to run migrations", zap.Error(err)) + } +} diff --git a/exporter/clickhouselogsexporter/exporter.go b/exporter/clickhouselogsexporter/exporter.go index 989fc65e..91bbb6a7 100644 --- a/exporter/clickhouselogsexporter/exporter.go +++ b/exporter/clickhouselogsexporter/exporter.go @@ -21,7 +21,6 @@ import ( "fmt" "log" "net/url" - "os" "strings" "sync" "time" @@ -30,9 +29,6 @@ import ( driver "github.com/ClickHouse/clickhouse-go/v2/lib/driver" "github.com/SigNoz/signoz-otel-collector/usage" "github.com/SigNoz/signoz-otel-collector/utils" - "github.com/golang-migrate/migrate/v4" - _ "github.com/golang-migrate/migrate/v4/database/clickhouse" - _ "github.com/golang-migrate/migrate/v4/source/file" "github.com/segmentio/ksuid" "go.opencensus.io/stats" "go.opencensus.io/stats/view" @@ -428,88 +424,9 @@ func newClickhouseClient(logger *zap.Logger, cfg *Config) (clickhouse.Conn, erro if err := db.Ping(ctx); err != nil { return nil, err } - - q := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s ON CLUSTER %s;", databaseName, CLUSTER) - err = db.Exec(ctx, q) - if err != nil { - return nil, fmt.Errorf("failed to create database, err: %s", err) - } - - // drop schema migrations table if running in docker multi node cluster mode so that migrations are run on new nodes - if cfg.DockerMultiNodeCluster { - err = dropSchemaMigrationsTable(db) - if err != nil { - logger.Error("Error dropping schema_migrations table", zap.Error(err)) - return nil, err - } - } - - // do the migration here - - // get the migrations folder - mgsFolder := os.Getenv("LOG_MIGRATIONS_FOLDER") - if mgsFolder == "" { - mgsFolder = migrationsFolder - } - - logger.Info("Running migrations from path: ", zap.Any("test", mgsFolder)) - clickhouseUrl, err := buildClickhouseMigrateURL(cfg) - if err != nil { - return nil, fmt.Errorf("failed to build Clickhouse migrate URL, error: %s", err) - } - m, err := migrate.New("file://"+mgsFolder, clickhouseUrl) - if err != nil { - return nil, fmt.Errorf("clickhouse Migrate failed to run, error: %s", err) - } - // enable migration file templating - m.EnableTemplating = true - - // run migrations - err = m.Up() - if err != nil && !strings.HasSuffix(err.Error(), "no change") { - return nil, fmt.Errorf("clickhouse Migrate failed to run, error: %s", err) - } - - logger.Info("Clickhouse Migrate finished") return db, nil } -func dropSchemaMigrationsTable(db clickhouse.Conn) error { - err := db.Exec(context.Background(), fmt.Sprintf(`DROP TABLE IF EXISTS %s.%s ON CLUSTER %s;`, - databaseName, "schema_migrations", CLUSTER)) - if err != nil { - return fmt.Errorf("error dropping schema_migrations table: %v", err) - } - return nil -} - -func buildClickhouseMigrateURL(cfg *Config) (string, error) { - // return fmt.Sprintf("clickhouse://localhost:9000?database=default&x-multi-statement=true"), nil - var clickhouseUrl string - parsedURL, err := url.Parse(cfg.DSN) - if err != nil { - return "", err - } - host := parsedURL.Host - if host == "" { - return "", fmt.Errorf("unable to parse host") - - } - paramMap, err := url.ParseQuery(parsedURL.RawQuery) - if err != nil { - return "", err - } - username := paramMap["username"] - password := paramMap["password"] - - if len(username) > 0 && len(password) > 0 { - clickhouseUrl = fmt.Sprintf("clickhouse://%s:%s@%s/%s?x-multi-statement=true&x-cluster-name=%s&x-migrations-table=schema_migrations&x-migrations-table-engine=MergeTree", username[0], password[0], host, databaseName, CLUSTER) - } else { - clickhouseUrl = fmt.Sprintf("clickhouse://%s/%s?x-multi-statement=true&x-cluster-name=%s&x-migrations-table=schema_migrations&x-migrations-table-engine=MergeTree", host, databaseName, CLUSTER) - } - return clickhouseUrl, nil -} - func renderInsertLogsSQL(cfg *Config) string { return fmt.Sprintf(insertLogsSQLTemplate, databaseName, DISTRIBUTED_LOGS_TABLE) } diff --git a/exporter/clickhouselogsexporter/migrations/000002_add_minmax_idx.down.sql b/exporter/clickhouselogsexporter/migrations/000002_add_minmax_idx.down.sql deleted file mode 100644 index 167455da..00000000 --- a/exporter/clickhouselogsexporter/migrations/000002_add_minmax_idx.down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE signoz_logs.logs ON CLUSTER cluster DROP INDEX IF EXISTS id_minmax; diff --git a/exporter/clickhouselogsexporter/migrations/000002_add_minmax_idx.up.sql b/exporter/clickhouselogsexporter/migrations/000002_add_minmax_idx.up.sql deleted file mode 100644 index 1735a69e..00000000 --- a/exporter/clickhouselogsexporter/migrations/000002_add_minmax_idx.up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE signoz_logs.logs ON CLUSTER cluster ADD INDEX IF NOT EXISTS id_minmax id TYPE minmax GRANULARITY 1; diff --git a/exporter/clickhouselogsexporter/migrations/000003_add_distributed_table.down.sql b/exporter/clickhouselogsexporter/migrations/000003_add_distributed_table.down.sql deleted file mode 100644 index c668e3fa..00000000 --- a/exporter/clickhouselogsexporter/migrations/000003_add_distributed_table.down.sql +++ /dev/null @@ -1,3 +0,0 @@ -DROP TABLE IF EXISTS signoz_logs.distributed_logs ON CLUSTER cluster; -DROP TABLE IF EXISTS signoz_logs.distributed_logs_atrribute_keys ON CLUSTER cluster; -DROP TABLE IF EXISTS signoz_logs.distributed_logs_resource_keys ON CLUSTER cluster; \ No newline at end of file diff --git a/exporter/clickhouselogsexporter/migrations/000004_ttl_only_drop_parts.down.sql b/exporter/clickhouselogsexporter/migrations/000004_ttl_only_drop_parts.down.sql deleted file mode 100644 index 6afaddfa..00000000 --- a/exporter/clickhouselogsexporter/migrations/000004_ttl_only_drop_parts.down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE signoz_logs.logs ON CLUSTER cluster MODIFY SETTING ttl_only_drop_parts = 0; diff --git a/exporter/clickhouselogsexporter/migrations/000004_ttl_only_drop_parts.up.sql b/exporter/clickhouselogsexporter/migrations/000004_ttl_only_drop_parts.up.sql deleted file mode 100644 index 2d62d107..00000000 --- a/exporter/clickhouselogsexporter/migrations/000004_ttl_only_drop_parts.up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE signoz_logs.logs ON CLUSTER cluster MODIFY SETTING ttl_only_drop_parts = 1; diff --git a/exporter/clickhouselogsexporter/migrations/000006_tag_attributes.down.sql b/exporter/clickhouselogsexporter/migrations/000006_tag_attributes.down.sql deleted file mode 100644 index 42982835..00000000 --- a/exporter/clickhouselogsexporter/migrations/000006_tag_attributes.down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE IF EXISTS signoz_logs.distributed_tag_attributes ON CLUSTER cluster; -DROP TABLE IF EXISTS signoz_logs.tag_attributes ON CLUSTER cluster; \ No newline at end of file diff --git a/exporter/clickhouselogsexporter/migrations/000007_default_indexes.down.sql b/exporter/clickhouselogsexporter/migrations/000007_default_indexes.down.sql deleted file mode 100644 index dd21eb16..00000000 --- a/exporter/clickhouselogsexporter/migrations/000007_default_indexes.down.sql +++ /dev/null @@ -1,5 +0,0 @@ -ALTER TABLE signoz_logs.logs ON CLUSTER cluster drop index IF EXISTS severity_number_idx; - -ALTER TABLE signoz_logs.logs ON CLUSTER cluster drop index IF EXISTS severity_text_idx; - -ALTER TABLE signoz_logs.logs ON CLUSTER cluster drop index IF EXISTS trace_flags_idx; \ No newline at end of file diff --git a/exporter/clickhousemetricsexporter/clickhouse.go b/exporter/clickhousemetricsexporter/clickhouse.go index 6a1de323..8b3af125 100644 --- a/exporter/clickhousemetricsexporter/clickhouse.go +++ b/exporter/clickhousemetricsexporter/clickhouse.go @@ -94,85 +94,6 @@ func NewClickHouse(params *ClickHouseParams) (base.Storage, error) { return nil, fmt.Errorf("database should be set in ClickHouse DSN") } - var queries []string - if params.DropDatabase { - queries = append(queries, fmt.Sprintf(`DROP DATABASE IF EXISTS %s ON CLUSTER %s;`, database, CLUSTER)) - } - queries = append(queries, fmt.Sprintf(`CREATE DATABASE IF NOT EXISTS %s ON CLUSTER %s`, database, CLUSTER)) - - queries = append(queries, fmt.Sprintf(` - CREATE TABLE IF NOT EXISTS %s.%s ON CLUSTER %s ( - metric_name LowCardinality(String), - fingerprint UInt64 Codec(DoubleDelta, LZ4), - timestamp_ms Int64 Codec(DoubleDelta, LZ4), - value Float64 Codec(Gorilla, LZ4) - ) - ENGINE = MergeTree - PARTITION BY toDate(timestamp_ms / 1000) - ORDER BY (metric_name, fingerprint, timestamp_ms) - TTL toDateTime(timestamp_ms/1000) + INTERVAL 2592000 SECOND DELETE;`, database, SAMPLES_TABLE, CLUSTER)) - - queries = append(queries, fmt.Sprintf(` - CREATE TABLE IF NOT EXISTS %s.%s ON CLUSTER %s AS %s.%s ENGINE = Distributed("%s", "%s", %s, cityHash64(metric_name, fingerprint));`, database, DISTRIBUTED_SAMPLES_TABLE, CLUSTER, database, SAMPLES_TABLE, CLUSTER, database, SAMPLES_TABLE)) - - queries = append(queries, fmt.Sprintf(` - ALTER TABLE %s.%s ON CLUSTER %s MODIFY SETTING ttl_only_drop_parts = 1;`, database, SAMPLES_TABLE, CLUSTER)) - - queries = append(queries, `SET allow_experimental_object_type = 1`) - - queries = append(queries, fmt.Sprintf(` - CREATE TABLE IF NOT EXISTS %s.%s ON CLUSTER %s( - metric_name LowCardinality(String), - fingerprint UInt64 Codec(DoubleDelta, LZ4), - timestamp_ms Int64 Codec(DoubleDelta, LZ4), - labels String Codec(ZSTD(5)) - ) - ENGINE = ReplacingMergeTree - PARTITION BY toDate(timestamp_ms / 1000) - ORDER BY (metric_name, fingerprint) - TTL toDateTime(timestamp_ms/1000) + INTERVAL 2592000 SECOND DELETE;`, database, TIME_SERIES_TABLE, CLUSTER)) - - queries = append(queries, fmt.Sprintf(` - CREATE TABLE IF NOT EXISTS %s.%s ON CLUSTER %s AS %s.%s ENGINE = Distributed("%s", %s, %s, cityHash64(metric_name, fingerprint));`, database, DISTRIBUTED_TIME_SERIES_TABLE, CLUSTER, database, TIME_SERIES_TABLE, CLUSTER, database, TIME_SERIES_TABLE)) - - queries = append(queries, fmt.Sprintf(` - ALTER TABLE %s.%s ON CLUSTER %s DROP COLUMN IF EXISTS labels_object`, database, TIME_SERIES_TABLE, CLUSTER)) - - queries = append(queries, fmt.Sprintf(` - ALTER TABLE %s.%s ON CLUSTER %s DROP COLUMN IF EXISTS labels_object`, database, DISTRIBUTED_TIME_SERIES_TABLE, CLUSTER)) - - queries = append(queries, fmt.Sprintf(` - ALTER TABLE %s.%s ON CLUSTER %s MODIFY SETTING ttl_only_drop_parts = 1;`, database, TIME_SERIES_TABLE, CLUSTER)) - - // Add temporality column to time_series table - queries = append(queries, fmt.Sprintf(` - ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS temporality LowCardinality(String) DEFAULT 'Unspecified' CODEC(ZSTD(5))`, database, TIME_SERIES_TABLE, CLUSTER)) - - queries = append(queries, fmt.Sprintf(` - ALTER TABLE %s.%s ON CLUSTER %s ADD COLUMN IF NOT EXISTS temporality LowCardinality(String) DEFAULT 'Unspecified' CODEC(ZSTD(5))`, database, DISTRIBUTED_TIME_SERIES_TABLE, CLUSTER)) - - // Add set index on temporality column - queries = append(queries, fmt.Sprintf(` - ALTER TABLE %s.%s ON CLUSTER %s ADD INDEX IF NOT EXISTS temporality_index temporality TYPE SET(3) GRANULARITY 1`, database, TIME_SERIES_TABLE, CLUSTER)) - - // Create a new table - queries = append(queries, fmt.Sprintf(` - CREATE TABLE IF NOT EXISTS %s.%s ON CLUSTER %s ( - env LowCardinality(String) DEFAULT 'default', - temporality LowCardinality(String) DEFAULT 'Unspecified', - metric_name LowCardinality(String), - fingerprint UInt64 CODEC(Delta, ZSTD), - timestamp_ms Int64 CODEC(Delta, ZSTD), - labels String CODEC(ZSTD(5)) - ) - ENGINE = ReplacingMergeTree - PARTITION BY toDate(timestamp_ms / 1000) - ORDER BY (env, temporality, metric_name, fingerprint);`, database, TIME_SERIES_TABLE_V3, CLUSTER)) - - // Create a new distributed table - queries = append(queries, fmt.Sprintf(` - CREATE TABLE IF NOT EXISTS %s.%s ON CLUSTER %s AS %s.%s ENGINE = Distributed("%s", %s, %s, cityHash64(env, temporality, metric_name, fingerprint));`, database, DISTRIBUTED_TIME_SERIES_TABLE_V3, CLUSTER, database, TIME_SERIES_TABLE_V3, CLUSTER, database, TIME_SERIES_TABLE_V3)) - options := &clickhouse.Options{ Addr: []string{dsnURL.Host}, } @@ -186,28 +107,10 @@ func NewClickHouse(params *ClickHouseParams) (base.Storage, error) { options.Auth = auth } conn, err := clickhouse.Open(options) - if err != nil { return nil, fmt.Errorf("could not connect to clickhouse: %s", err) } - for _, q := range queries { - q = strings.TrimSpace(q) - l.Infof("Executing:\n%s\n", q) - if err = conn.Exec(context.Background(), q); err != nil { - return nil, err - } - } - - // TODO(srikanthccv): Remove this once we have a better way to handle data and last write - removeTTL := fmt.Sprintf(` - ALTER TABLE %s.%s ON CLUSTER %s REMOVE TTL;`, database, TIME_SERIES_TABLE, CLUSTER) - if err = conn.Exec(context.Background(), removeTTL); err != nil { - if !strings.Contains(err.Error(), "Table doesn't have any table TTL expression, cannot remove.") { - return nil, err - } - } - ch := &clickHouse{ conn: conn, l: l, diff --git a/exporter/clickhousetracesexporter/clickhouse_factory.go b/exporter/clickhousetracesexporter/clickhouse_factory.go index 8eff7e99..e9d9c8da 100644 --- a/exporter/clickhousetracesexporter/clickhouse_factory.go +++ b/exporter/clickhousetracesexporter/clickhouse_factory.go @@ -15,16 +15,10 @@ package clickhousetracesexporter import ( - "context" "flag" "fmt" - "net/url" - "strings" "github.com/ClickHouse/clickhouse-go/v2" - "github.com/golang-migrate/migrate/v4" - _ "github.com/golang-migrate/migrate/v4/database/clickhouse" - _ "github.com/golang-migrate/migrate/v4/source/file" "github.com/spf13/viper" "go.opencensus.io/stats" "go.opencensus.io/stats/view" @@ -99,167 +93,9 @@ func (f *Factory) Initialize(logger *zap.Logger) error { f.archive = archive } - - err = patchGroupByParenInMV(db, f) - if err != nil { - return err - } - - // drop schema migrations table if running in docker multi node cluster mode so that migrations are run on new nodes - if f.Options.primary.DockerMultiNodeCluster { - err = dropSchemaMigrationsTable(db, f) - if err != nil { - return err - } - } - - f.logger.Info("Running migrations from path: ", zap.Any("test", f.Options.primary.Migrations)) - clickhouseUrl, err := buildClickhouseMigrateURL(f.Options.primary.Datasource, f.Options.primary.Cluster) - if err != nil { - return fmt.Errorf("Failed to build Clickhouse migrate URL, error: %s", err) - } - m, err := migrate.New( - "file://"+f.Options.primary.Migrations, - clickhouseUrl) - if err != nil { - return fmt.Errorf("Clickhouse Migrate failed to run, error: %s", err) - } - // enable migration file templating - m.EnableTemplating = true - - // run migrations - err = m.Up() - if err != nil && !strings.HasSuffix(err.Error(), "no change") { - f.logger.Error("clickhouse Migrate failed to run", zap.Error(err)) - return fmt.Errorf("clickhouse Migrate failed to run, error: %s", err) - } - - f.logger.Info("Clickhouse Migrate finished") return nil } -func patchGroupByParenInMV(db clickhouse.Conn, f *Factory) error { - - // check if views already exist, if not, skip as patch is not required for fresh install - for _, table := range []string{f.Options.getPrimary().DependencyGraphDbMV, f.Options.getPrimary().DependencyGraphServiceMV, f.Options.getPrimary().DependencyGraphMessagingMV} { - var exists uint8 - err := db.QueryRow(context.Background(), fmt.Sprintf("EXISTS VIEW %s.%s", f.Options.getPrimary().TraceDatabase, table)).Scan(&exists) - if err != nil { - return err - } - if exists == 0 { - f.logger.Info("View does not exist, skipping patch", zap.String("table", table)) - return nil - } - } - f.logger.Info("Patching views") - // drop views - for _, table := range []string{f.Options.getPrimary().DependencyGraphDbMV, f.Options.getPrimary().DependencyGraphServiceMV, f.Options.getPrimary().DependencyGraphMessagingMV} { - err := db.Exec(context.Background(), fmt.Sprintf("DROP VIEW IF EXISTS %s.%s ON CLUSTER %s", f.Options.getPrimary().TraceDatabase, table, f.Options.getPrimary().Cluster)) - if err != nil { - f.logger.Error(fmt.Sprintf("Error dropping %s view", table), zap.Error(err)) - return fmt.Errorf("error dropping %s view: %v", table, err) - } - } - - // create views with patched group by - err := db.Exec(context.Background(), fmt.Sprintf(`CREATE MATERIALIZED VIEW IF NOT EXISTS %s.%s ON CLUSTER %s - TO %s.%s AS - SELECT - A.serviceName as src, - B.serviceName as dest, - quantilesState(0.5, 0.75, 0.9, 0.95, 0.99)(toFloat64(B.durationNano)) as duration_quantiles_state, - countIf(B.statusCode=2) as error_count, - count(*) as total_count, - toStartOfMinute(B.timestamp) as timestamp - FROM %s.%s AS A, %s.%s AS B - WHERE (A.serviceName != B.serviceName) AND (A.spanID = B.parentSpanID) - GROUP BY timestamp, src, dest;`, f.Options.getPrimary().TraceDatabase, f.Options.getPrimary().DependencyGraphServiceMV, - f.Options.getPrimary().Cluster, f.Options.getPrimary().TraceDatabase, f.Options.getPrimary().DependencyGraphTable, - f.Options.getPrimary().TraceDatabase, f.Options.getPrimary().LocalIndexTable, f.Options.getPrimary().TraceDatabase, - f.Options.getPrimary().LocalIndexTable)) - if err != nil { - f.logger.Error("Error creating "+f.Options.getPrimary().DependencyGraphServiceMV, zap.Error(err)) - return fmt.Errorf("error creating %s: %v", f.Options.getPrimary().DependencyGraphServiceMV, err) - } - err = db.Exec(context.Background(), fmt.Sprintf(`CREATE MATERIALIZED VIEW IF NOT EXISTS %s.%s ON CLUSTER %s - TO %s.%s AS - SELECT - serviceName as src, - tagMap['db.system'] as dest, - quantilesState(0.5, 0.75, 0.9, 0.95, 0.99)(toFloat64(durationNano)) as duration_quantiles_state, - countIf(statusCode=2) as error_count, - count(*) as total_count, - toStartOfMinute(timestamp) as timestamp - FROM %s.%s - WHERE dest != '' and kind != 2 - GROUP BY timestamp, src, dest;`, f.Options.getPrimary().TraceDatabase, f.Options.getPrimary().DependencyGraphDbMV, - f.Options.getPrimary().Cluster, f.Options.getPrimary().TraceDatabase, f.Options.getPrimary().DependencyGraphTable, - f.Options.getPrimary().TraceDatabase, f.Options.getPrimary().LocalIndexTable)) - if err != nil { - f.logger.Error("Error creating "+f.Options.getPrimary().DependencyGraphDbMV, zap.Error(err)) - return fmt.Errorf("error creating %s: %v", f.Options.getPrimary().DependencyGraphDbMV, err) - } - err = db.Exec(context.Background(), fmt.Sprintf(`CREATE MATERIALIZED VIEW IF NOT EXISTS %s.%s ON CLUSTER %s - TO %s.%s AS - SELECT - serviceName as src, - tagMap['messaging.system'] as dest, - quantilesState(0.5, 0.75, 0.9, 0.95, 0.99)(toFloat64(durationNano)) as duration_quantiles_state, - countIf(statusCode=2) as error_count, - count(*) as total_count, - toStartOfMinute(timestamp) as timestamp - FROM %s.%s - WHERE dest != '' and kind != 2 - GROUP BY timestamp, src, dest;`, f.Options.getPrimary().TraceDatabase, f.Options.getPrimary().DependencyGraphMessagingMV, - f.Options.getPrimary().Cluster, f.Options.getPrimary().TraceDatabase, f.Options.getPrimary().DependencyGraphTable, - f.Options.getPrimary().TraceDatabase, f.Options.getPrimary().LocalIndexTable)) - if err != nil { - f.logger.Error("Error creating "+f.Options.getPrimary().DependencyGraphMessagingMV, zap.Error(err)) - return fmt.Errorf("error creating %s: %v", f.Options.getPrimary().DependencyGraphMessagingMV, err) - } - - return nil -} - -func dropSchemaMigrationsTable(db clickhouse.Conn, f *Factory) error { - err := db.Exec(context.Background(), fmt.Sprintf(`DROP TABLE IF EXISTS %s.%s ON CLUSTER %s;`, - f.Options.getPrimary().TraceDatabase, "schema_migrations", f.Options.getPrimary().Cluster)) - if err != nil { - f.logger.Error("Error dropping schema_migrations table", zap.Error(err)) - return fmt.Errorf("error dropping schema_migrations table: %v", err) - } - return nil -} - -func buildClickhouseMigrateURL(datasource string, cluster string) (string, error) { - // return fmt.Sprintf("clickhouse://localhost:9000?database=default&x-multi-statement=true"), nil - var clickhouseUrl string - database := "signoz_traces" - parsedURL, err := url.Parse(datasource) - if err != nil { - return "", err - } - host := parsedURL.Host - if host == "" { - return "", fmt.Errorf("Unable to parse host") - - } - paramMap, err := url.ParseQuery(parsedURL.RawQuery) - if err != nil { - return "", err - } - username := paramMap["username"] - password := paramMap["password"] - - if len(username) > 0 && len(password) > 0 { - clickhouseUrl = fmt.Sprintf("clickhouse://%s:%s@%s/%s?x-multi-statement=true&x-cluster-name=%s&x-migrations-table=schema_migrations&x-migrations-table-engine=MergeTree", username[0], password[0], host, database, cluster) - } else { - clickhouseUrl = fmt.Sprintf("clickhouse://%s/%s?x-multi-statement=true&x-cluster-name=%s&x-migrations-table=schema_migrations&x-migrations-table-engine=MergeTree", host, database, cluster) - } - return clickhouseUrl, nil -} - func (f *Factory) connect(cfg *namespaceConfig) (clickhouse.Conn, error) { if cfg.Encoding != EncodingJSON && cfg.Encoding != EncodingProto { return nil, fmt.Errorf("unknown encoding %q, supported: %q, %q", cfg.Encoding, EncodingJSON, EncodingProto) diff --git a/exporter/clickhousetracesexporter/migrations/000001_init_db.down.sql b/exporter/clickhousetracesexporter/migrations/000001_init_db.down.sql deleted file mode 100644 index 9f110564..00000000 --- a/exporter/clickhousetracesexporter/migrations/000001_init_db.down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE IF EXISTS signoz_traces.signoz_index ON CLUSTER cluster; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000002_init_schema.down.sql b/exporter/clickhousetracesexporter/migrations/000002_init_schema.down.sql deleted file mode 100644 index f6eda191..00000000 --- a/exporter/clickhousetracesexporter/migrations/000002_init_schema.down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE signoz_traces.signoz_index ON CLUSTER cluster DROP COLUMN IF EXISTS events; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000002_init_schema.up.sql b/exporter/clickhousetracesexporter/migrations/000002_init_schema.up.sql deleted file mode 100644 index 0e53ffb2..00000000 --- a/exporter/clickhousetracesexporter/migrations/000002_init_schema.up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE signoz_traces.signoz_index ON CLUSTER cluster ADD COLUMN IF NOT EXISTS events Array(String); \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000004_update_table.down.sql b/exporter/clickhousetracesexporter/migrations/000004_update_table.down.sql deleted file mode 100644 index 209de8e6..00000000 --- a/exporter/clickhousetracesexporter/migrations/000004_update_table.down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE signoz_traces.signoz_index ON CLUSTER cluster DROP COLUMN IF EXISTS httpMethod, DROP COLUMN IF EXISTS httpUrl, DROP COLUMN IF EXISTS httpCode, DROP COLUMN IF EXISTS httpRoute, DROP COLUMN IF EXISTS httpHost, DROP COLUMN IF EXISTS msgSystem, DROP COLUMN IF EXISTS msgOperation; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000004_update_table.up.sql b/exporter/clickhousetracesexporter/migrations/000004_update_table.up.sql deleted file mode 100644 index ea7b142a..00000000 --- a/exporter/clickhousetracesexporter/migrations/000004_update_table.up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE signoz_traces.signoz_index ON CLUSTER cluster ADD COLUMN IF NOT EXISTS httpMethod LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS httpUrl LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS httpCode LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS httpRoute LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS httpHost LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS msgSystem LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS msgOperation LowCardinality(String) CODEC(ZSTD(1)); diff --git a/exporter/clickhousetracesexporter/migrations/000005_add_hasError.down.sql b/exporter/clickhousetracesexporter/migrations/000005_add_hasError.down.sql deleted file mode 100644 index a4c98e70..00000000 --- a/exporter/clickhousetracesexporter/migrations/000005_add_hasError.down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE signoz_traces.signoz_index ON CLUSTER cluster DROP COLUMN IF EXISTS hasError; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000005_add_hasError.up.sql b/exporter/clickhousetracesexporter/migrations/000005_add_hasError.up.sql deleted file mode 100644 index 95cc9bbf..00000000 --- a/exporter/clickhousetracesexporter/migrations/000005_add_hasError.up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE signoz_traces.signoz_index ON CLUSTER cluster ADD COLUMN IF NOT EXISTS hasError Int32; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000006_add_tag_map.down.sql b/exporter/clickhousetracesexporter/migrations/000006_add_tag_map.down.sql deleted file mode 100644 index 62a9cb36..00000000 --- a/exporter/clickhousetracesexporter/migrations/000006_add_tag_map.down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE signoz_traces.signoz_index ON CLUSTER cluster DROP COLUMN IF EXISTS tagMap; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000006_add_tag_map.up.sql b/exporter/clickhousetracesexporter/migrations/000006_add_tag_map.up.sql deleted file mode 100644 index d9a77ae3..00000000 --- a/exporter/clickhousetracesexporter/migrations/000006_add_tag_map.up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE signoz_traces.signoz_index ON CLUSTER cluster ADD COLUMN IF NOT EXISTS tagMap Map(LowCardinality(String), String); \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000007_init_signoz_index_v2.down.sql b/exporter/clickhousetracesexporter/migrations/000007_init_signoz_index_v2.down.sql deleted file mode 100644 index 11a6be94..00000000 --- a/exporter/clickhousetracesexporter/migrations/000007_init_signoz_index_v2.down.sql +++ /dev/null @@ -1,3 +0,0 @@ -DROP TABLE IF EXISTS signoz_traces.signoz_index_v2 ON CLUSTER cluster; - -SET allow_experimental_projection_optimization = 0; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000008_init_signoz_spans.down.sql b/exporter/clickhousetracesexporter/migrations/000008_init_signoz_spans.down.sql deleted file mode 100644 index 16ec896a..00000000 --- a/exporter/clickhousetracesexporter/migrations/000008_init_signoz_spans.down.sql +++ /dev/null @@ -1 +0,0 @@ -DROP TABLE IF EXISTS signoz_traces.signoz_spans ON CLUSTER cluster; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000010_add_gRPC_code_method.down.sql b/exporter/clickhousetracesexporter/migrations/000010_add_gRPC_code_method.down.sql deleted file mode 100644 index cf151a0a..00000000 --- a/exporter/clickhousetracesexporter/migrations/000010_add_gRPC_code_method.down.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER cluster DROP COLUMN IF EXISTS gRPCCode, DROP COLUMN IF EXISTS gRPCMethod; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000010_add_gRPC_code_method.up.sql b/exporter/clickhousetracesexporter/migrations/000010_add_gRPC_code_method.up.sql deleted file mode 100644 index 07ccc7d0..00000000 --- a/exporter/clickhousetracesexporter/migrations/000010_add_gRPC_code_method.up.sql +++ /dev/null @@ -1 +0,0 @@ -ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER cluster ADD COLUMN IF NOT EXISTS gRPCMethod LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS gRPCCode LowCardinality(String) CODEC(ZSTD(1)); \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000011_init_durationMV.down.sql b/exporter/clickhousetracesexporter/migrations/000011_init_durationMV.down.sql deleted file mode 100644 index c03d485c..00000000 --- a/exporter/clickhousetracesexporter/migrations/000011_init_durationMV.down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE IF EXISTS signoz_traces.durationSort ON CLUSTER cluster; -DROP VIEW IF EXISTS signoz_traces.durationSortMV ON CLUSTER cluster; diff --git a/exporter/clickhousetracesexporter/migrations/000014_add_usage_explorer_mv.down.sql b/exporter/clickhousetracesexporter/migrations/000014_add_usage_explorer_mv.down.sql deleted file mode 100644 index a0fcf525..00000000 --- a/exporter/clickhousetracesexporter/migrations/000014_add_usage_explorer_mv.down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE IF EXISTS signoz_traces.usage_explorer ON CLUSTER cluster; -DROP VIEW IF EXISTS signoz_traces.usage_explorer_mv ON CLUSTER cluster; diff --git a/exporter/clickhousetracesexporter/migrations/000019_ttl_only_drop_parts.down.sql b/exporter/clickhousetracesexporter/migrations/000019_ttl_only_drop_parts.down.sql deleted file mode 100644 index 29313d16..00000000 --- a/exporter/clickhousetracesexporter/migrations/000019_ttl_only_drop_parts.down.sql +++ /dev/null @@ -1,6 +0,0 @@ -ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER cluster MODIFY SETTING ttl_only_drop_parts = 0; -ALTER TABLE signoz_traces.signoz_error_index_v2 ON CLUSTER cluster MODIFY SETTING ttl_only_drop_parts = 0; -ALTER TABLE signoz_traces.signoz_spans ON CLUSTER cluster MODIFY SETTING ttl_only_drop_parts = 0; -ALTER TABLE signoz_traces.durationSort ON CLUSTER cluster MODIFY SETTING ttl_only_drop_parts = 0; -ALTER TABLE signoz_traces.dependency_graph_minutes ON CLUSTER cluster MODIFY SETTING ttl_only_drop_parts = 0; -ALTER TABLE signoz_traces.usage_explorer ON CLUSTER cluster MODIFY SETTING ttl_only_drop_parts = 0; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000019_ttl_only_drop_parts.up.sql b/exporter/clickhousetracesexporter/migrations/000019_ttl_only_drop_parts.up.sql deleted file mode 100644 index 8438ed90..00000000 --- a/exporter/clickhousetracesexporter/migrations/000019_ttl_only_drop_parts.up.sql +++ /dev/null @@ -1,6 +0,0 @@ -ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER cluster MODIFY SETTING ttl_only_drop_parts = 1; -ALTER TABLE signoz_traces.signoz_error_index_v2 ON CLUSTER cluster MODIFY SETTING ttl_only_drop_parts = 1; -ALTER TABLE signoz_traces.signoz_spans ON CLUSTER cluster MODIFY SETTING ttl_only_drop_parts = 1; -ALTER TABLE signoz_traces.durationSort ON CLUSTER cluster MODIFY SETTING ttl_only_drop_parts = 1; -ALTER TABLE signoz_traces.dependency_graph_minutes ON CLUSTER cluster MODIFY SETTING ttl_only_drop_parts = 1; -ALTER TABLE signoz_traces.usage_explorer ON CLUSTER cluster MODIFY SETTING ttl_only_drop_parts = 1; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000023_span_attributes_mv.down.sql b/exporter/clickhousetracesexporter/migrations/000023_span_attributes_mv.down.sql deleted file mode 100644 index cb5b3c71..00000000 --- a/exporter/clickhousetracesexporter/migrations/000023_span_attributes_mv.down.sql +++ /dev/null @@ -1,2 +0,0 @@ -DROP TABLE IF EXISTS signoz_traces.distributed_span_attributes ON CLUSTER cluster -DROP TABLE IF EXISTS signoz_traces.span_attributes ON CLUSTER cluster; diff --git a/migrator/manager.go b/migrator/manager.go new file mode 100644 index 00000000..db8f47f0 --- /dev/null +++ b/migrator/manager.go @@ -0,0 +1,86 @@ +package migrator + +import ( + "context" + "fmt" + + "github.com/SigNoz/signoz-otel-collector/migrator/migrators" + basemigrator "github.com/SigNoz/signoz-otel-collector/migrator/migrators/baseMigrator" + "github.com/SigNoz/signoz-otel-collector/migrator/migrators/logs" + "github.com/SigNoz/signoz-otel-collector/migrator/migrators/metrics" + "github.com/SigNoz/signoz-otel-collector/migrator/migrators/traces" + "go.uber.org/zap" +) + +type MigrationManager struct { + Migrators []migrators.Migrator + logger *zap.Logger +} + +func NewMigrationManager(dsn string, clusterName string, isMultiNodeCluster bool) (*MigrationManager, error) { + logger := zap.L().With(zap.String("component", "migrationManager")) + migratorConfig := migrators.MigratorConfig{ + DSN: dsn, + ClusterName: clusterName, + IsMultiNodeCluster: isMultiNodeCluster, + } + + logsMigrator, err := createNewMigrator("logs", migratorConfig) + if err != nil { + logger.Error("Failed to create logs migrator", zap.Error(err)) + return nil, err + } + metricsMigrator, err := createNewMigrator("metrics", migratorConfig) + if err != nil { + logger.Error("Failed to create metrics migrator", zap.Error(err)) + return nil, err + } + tracesMigrator, err := createNewMigrator("traces", migratorConfig) + if err != nil { + logger.Error("Failed to create traces migrator", zap.Error(err)) + return nil, err + } + return &MigrationManager{ + Migrators: []migrators.Migrator{ + logsMigrator, + metricsMigrator, + tracesMigrator, + }, + logger: logger, + }, nil +} + +func createNewMigrator(migratorType string, cfg migrators.MigratorConfig) (migrators.Migrator, error) { + logger := zap.L().With(zap.String("migrator", migratorType)) + b, err := basemigrator.New(cfg, logger) + if err != nil { + logger.Error("Failed to create base migrator", zap.Error(err)) + return nil, err + } + switch migratorType { + case "traces": + return &traces.TracesMigrator{BaseMigrator: b}, nil + case "metrics": + return &metrics.MetricsMigrator{BaseMigrator: b}, nil + case "logs": + return &logs.LogsMigrator{BaseMigrator: b}, nil + default: + return nil, fmt.Errorf("invalid migrator type: %s", migratorType) + } +} + +func (m *MigrationManager) Migrate(ctx context.Context) error { + m.logger.Info("Running migrations for all migrators") + for _, migrator := range m.Migrators { + m.logger.Info(fmt.Sprintf("Running migrations for %s", migrator.Name()), zap.String("migrator", migrator.Name())) + err := migrator.Migrate(ctx) + if err != nil { + m.logger.Error("Failed to run migrations for migrator", zap.String("migrator", migrator.Name()), zap.Error(err)) + return err + } + } + m.logger.Info("Finished running migrations for all migrators") + return nil +} + + diff --git a/migrator/migrators/baseMigrator/migrator.go b/migrator/migrators/baseMigrator/migrator.go new file mode 100644 index 00000000..aa1e734d --- /dev/null +++ b/migrator/migrators/baseMigrator/migrator.go @@ -0,0 +1,141 @@ +package basemigrator + +import ( + "context" + "fmt" + "net/url" + "strings" + + "github.com/ClickHouse/clickhouse-go/v2" + "github.com/ClickHouse/clickhouse-go/v2/lib/driver" + "github.com/SigNoz/signoz-otel-collector/migrator/migrators" + "github.com/golang-migrate/migrate/v4" + _ "github.com/golang-migrate/migrate/v4/database/clickhouse" + _ "github.com/golang-migrate/migrate/v4/source/file" + "go.uber.org/zap" +) + +type BaseMigrator struct { + Cfg migrators.MigratorConfig + Logger *zap.Logger + DB driver.Conn +} + +func createClickhouseConnection(dsn string) (driver.Conn, error) { + dsnURL, err := url.Parse(dsn) + if err != nil { + return nil, fmt.Errorf("failed to parse dsn: %w", err) + } + options := &clickhouse.Options{ + Addr: []string{dsnURL.Host}, + } + if dsnURL.Query().Get("username") != "" { + auth := clickhouse.Auth{ + Username: dsnURL.Query().Get("username"), + Password: dsnURL.Query().Get("password"), + } + options.Auth = auth + } + db, err := clickhouse.Open(options) + if err != nil { + return nil, fmt.Errorf("failed to open clickhouse connection: %w", err) + } + + if err := db.Ping(context.Background()); err != nil { + return nil, fmt.Errorf("failed to ping clickhouse: %w", err) + } + + return db, nil +} + +func New(cfg migrators.MigratorConfig, logger *zap.Logger) (*BaseMigrator, error) { + dbConn, err := createClickhouseConnection(cfg.DSN) + if err != nil { + logger.Error("Failed to create clickhouse connection", zap.Error(err)) + return nil, err + } + + return &BaseMigrator{ + Cfg: cfg, + Logger: logger, + DB: dbConn, + }, nil +} + +func (m *BaseMigrator) buildClickhouseMigrateURL(database string) (string, error) { + var clickhouseUrl string + parsedURL, err := url.Parse(m.Cfg.DSN) + if err != nil { + return "", err + } + host := parsedURL.Host + if host == "" { + return "", fmt.Errorf("unable to parse host") + + } + paramMap, err := url.ParseQuery(parsedURL.RawQuery) + if err != nil { + return "", err + } + username := paramMap["username"] + password := paramMap["password"] + + if len(username) > 0 && len(password) > 0 { + clickhouseUrl = fmt.Sprintf("clickhouse://%s:%s@%s/%s?x-multi-statement=true&x-cluster-name=%s&x-migrations-table=schema_migrations&x-migrations-table-engine=MergeTree", username[0], password[0], host, database, m.Cfg.ClusterName) + } else { + clickhouseUrl = fmt.Sprintf("clickhouse://%s/%s?x-multi-statement=true&x-cluster-name=%s&x-migrations-table=schema_migrations&x-migrations-table-engine=MergeTree", host, database, m.Cfg.ClusterName) + } + return clickhouseUrl, nil +} + +func (m *BaseMigrator) CreateDB(ctx context.Context, database string) error { + q := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s ON CLUSTER %s;", database, m.Cfg.ClusterName) + err := m.DB.Exec(ctx, q) + if err != nil { + return fmt.Errorf("failed to create database, err: %s", err) + } + return nil +} + +func (m *BaseMigrator) DropSchemaMigrationsTable(ctx context.Context, database string) error { + err := m.DB.Exec(ctx, fmt.Sprintf(`DROP TABLE IF EXISTS %s.%s ON CLUSTER %s;`, database, "schema_migrations", m.Cfg.ClusterName)) + if err != nil { + return fmt.Errorf("error dropping schema_migrations table: %v", err) + } + return nil +} + +func (m *BaseMigrator) RunSqlMigrations(ctx context.Context, migrationFolder, database string) error { + clickhouseUrl, err := m.buildClickhouseMigrateURL(database) + if err != nil { + return fmt.Errorf("failed to build clickhouse migrate url, err: %s", err) + } + migrator, err := migrate.New("file://"+migrationFolder, clickhouseUrl) + if err != nil { + return fmt.Errorf("failed to create migrator, err: %s", err) + } + migrator.EnableTemplating = true + + err = migrator.Up() + if err != nil && !strings.HasSuffix(err.Error(), "no change") { + return fmt.Errorf("clickhouse migrate failed to run, error: %s", err) + } + return nil +} + +func (m *BaseMigrator) Migrate(ctx context.Context, database string, migrationFolder string) error { + err := m.CreateDB(ctx, database) + if err != nil { + return err + } + + // drop schema migrations table if running in docker multi node cluster mode so that migrations are run on new nodes + if m.Cfg.IsMultiNodeCluster { + err := m.DropSchemaMigrationsTable(ctx, database) + if err != nil { + return err + } + } + + return m.RunSqlMigrations(ctx, migrationFolder, database) +} diff --git a/exporter/clickhouselogsexporter/migrations/000001_init_db.down.sql b/migrator/migrators/logs/migrations/000001_init_db.down.sql similarity index 51% rename from exporter/clickhouselogsexporter/migrations/000001_init_db.down.sql rename to migrator/migrators/logs/migrations/000001_init_db.down.sql index cb0e21fc..701d29b9 100644 --- a/exporter/clickhouselogsexporter/migrations/000001_init_db.down.sql +++ b/migrator/migrators/logs/migrations/000001_init_db.down.sql @@ -1,7 +1,7 @@ -DROP TABLE IF EXISTS signoz_logs.logs ON CLUSTER cluster; -DROP TABLE IF EXISTS signoz_logs.logs_atrribute_keys ON CLUSTER cluster; -DROP TABLE IF EXISTS signoz_logs.logs_resource_keys ON CLUSTER cluster; -DROP TABLE IF EXISTS signoz_logs.resource_keys_string_final_mv ON CLUSTER cluster; -DROP TABLE IF EXISTS signoz_logs.atrribute_keys_float64_final_mv ON CLUSTER cluster; -DROP TABLE IF EXISTS signoz_logs.atrribute_keys_int64_final_mv ON CLUSTER cluster; -DROP TABLE IF EXISTS signoz_logs.atrribute_keys_string_final_mv ON CLUSTER cluster; \ No newline at end of file +DROP TABLE IF EXISTS signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP TABLE IF EXISTS signoz_logs.logs_atrribute_keys ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP TABLE IF EXISTS signoz_logs.logs_resource_keys ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP TABLE IF EXISTS signoz_logs.resource_keys_string_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP TABLE IF EXISTS signoz_logs.atrribute_keys_float64_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP TABLE IF EXISTS signoz_logs.atrribute_keys_int64_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP TABLE IF EXISTS signoz_logs.atrribute_keys_string_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; \ No newline at end of file diff --git a/exporter/clickhouselogsexporter/migrations/000001_init_db.up.sql b/migrator/migrators/logs/migrations/000001_init_db.up.sql similarity index 82% rename from exporter/clickhouselogsexporter/migrations/000001_init_db.up.sql rename to migrator/migrators/logs/migrations/000001_init_db.up.sql index e427d0cf..f2551900 100644 --- a/exporter/clickhouselogsexporter/migrations/000001_init_db.up.sql +++ b/migrator/migrators/logs/migrations/000001_init_db.up.sql @@ -1,5 +1,4 @@ --- https://altinity.com/blog/2019/7/new-encodings-to-improve-clickhouse -CREATE TABLE IF NOT EXISTS signoz_logs.logs ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ( timestamp UInt64 CODEC(DoubleDelta, LZ4), observed_timestamp UInt64 CODEC(DoubleDelta, LZ4), id String CODEC(ZSTD(1)), @@ -25,7 +24,7 @@ TTL toDateTime(timestamp / 1000000000) + INTERVAL 604800 SECOND DELETE; -CREATE TABLE IF NOT EXISTS signoz_logs.logs_atrribute_keys ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_logs.logs_atrribute_keys ON CLUSTER {{.SIGNOZ_CLUSTER}} ( name String, datatype String )ENGINE = ReplacingMergeTree @@ -33,32 +32,32 @@ ORDER BY (name, datatype); -CREATE TABLE IF NOT EXISTS signoz_logs.logs_resource_keys ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_logs.logs_resource_keys ON CLUSTER {{.SIGNOZ_CLUSTER}} ( name String, datatype String )ENGINE = ReplacingMergeTree ORDER BY (name, datatype); -CREATE MATERIALIZED VIEW IF NOT EXISTS atrribute_keys_string_final_mv ON CLUSTER cluster TO signoz_logs.logs_atrribute_keys AS +CREATE MATERIALIZED VIEW IF NOT EXISTS atrribute_keys_string_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_logs.logs_atrribute_keys AS SELECT distinct arrayJoin(attributes_string_key) as name, 'String' datatype FROM signoz_logs.logs ORDER BY name; -CREATE MATERIALIZED VIEW IF NOT EXISTS atrribute_keys_int64_final_mv ON CLUSTER cluster TO signoz_logs.logs_atrribute_keys AS +CREATE MATERIALIZED VIEW IF NOT EXISTS atrribute_keys_int64_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_logs.logs_atrribute_keys AS SELECT distinct arrayJoin(attributes_int64_key) as name, 'Int64' datatype FROM signoz_logs.logs ORDER BY name; -CREATE MATERIALIZED VIEW IF NOT EXISTS atrribute_keys_float64_final_mv ON CLUSTER cluster TO signoz_logs.logs_atrribute_keys AS +CREATE MATERIALIZED VIEW IF NOT EXISTS atrribute_keys_float64_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_logs.logs_atrribute_keys AS SELECT distinct arrayJoin(attributes_float64_key) as name, 'Float64' datatype FROM signoz_logs.logs ORDER BY name; -CREATE MATERIALIZED VIEW IF NOT EXISTS resource_keys_string_final_mv ON CLUSTER cluster TO signoz_logs.logs_resource_keys AS +CREATE MATERIALIZED VIEW IF NOT EXISTS resource_keys_string_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_logs.logs_resource_keys AS SELECT distinct arrayJoin(resources_string_key) as name, 'String' datatype FROM signoz_logs.logs diff --git a/migrator/migrators/logs/migrations/000002_add_minmax_idx.down.sql b/migrator/migrators/logs/migrations/000002_add_minmax_idx.down.sql new file mode 100644 index 00000000..ac8aa729 --- /dev/null +++ b/migrator/migrators/logs/migrations/000002_add_minmax_idx.down.sql @@ -0,0 +1 @@ +ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP INDEX IF EXISTS id_minmax; diff --git a/migrator/migrators/logs/migrations/000002_add_minmax_idx.up.sql b/migrator/migrators/logs/migrations/000002_add_minmax_idx.up.sql new file mode 100644 index 00000000..3fd329b3 --- /dev/null +++ b/migrator/migrators/logs/migrations/000002_add_minmax_idx.up.sql @@ -0,0 +1 @@ +ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD INDEX IF NOT EXISTS id_minmax id TYPE minmax GRANULARITY 1; diff --git a/migrator/migrators/logs/migrations/000003_add_distributed_table.down.sql b/migrator/migrators/logs/migrations/000003_add_distributed_table.down.sql new file mode 100644 index 00000000..d9f1ae21 --- /dev/null +++ b/migrator/migrators/logs/migrations/000003_add_distributed_table.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP TABLE IF EXISTS signoz_logs.distributed_logs_atrribute_keys ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP TABLE IF EXISTS signoz_logs.distributed_logs_resource_keys ON CLUSTER {{.SIGNOZ_CLUSTER}}; \ No newline at end of file diff --git a/exporter/clickhouselogsexporter/migrations/000003_add_distributed_table.up.sql b/migrator/migrators/logs/migrations/000003_add_distributed_table.up.sql similarity index 70% rename from exporter/clickhouselogsexporter/migrations/000003_add_distributed_table.up.sql rename to migrator/migrators/logs/migrations/000003_add_distributed_table.up.sql index 3839e53b..818da2b8 100644 --- a/exporter/clickhouselogsexporter/migrations/000003_add_distributed_table.up.sql +++ b/migrator/migrators/logs/migrations/000003_add_distributed_table.up.sql @@ -1,8 +1,8 @@ -CREATE TABLE IF NOT EXISTS signoz_logs.distributed_logs ON CLUSTER cluster AS signoz_logs.logs +CREATE TABLE IF NOT EXISTS signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} AS signoz_logs.logs ENGINE = Distributed("cluster", "signoz_logs", logs, cityHash64(id)); -CREATE TABLE IF NOT EXISTS signoz_logs.distributed_logs_atrribute_keys ON CLUSTER cluster AS signoz_logs.logs_atrribute_keys +CREATE TABLE IF NOT EXISTS signoz_logs.distributed_logs_atrribute_keys ON CLUSTER {{.SIGNOZ_CLUSTER}} AS signoz_logs.logs_atrribute_keys ENGINE = Distributed("cluster", "signoz_logs", logs_atrribute_keys, cityHash64(datatype)); -CREATE TABLE IF NOT EXISTS signoz_logs.distributed_logs_resource_keys ON CLUSTER cluster AS signoz_logs.logs_resource_keys +CREATE TABLE IF NOT EXISTS signoz_logs.distributed_logs_resource_keys ON CLUSTER {{.SIGNOZ_CLUSTER}} AS signoz_logs.logs_resource_keys ENGINE = Distributed("cluster", "signoz_logs", logs_resource_keys, cityHash64(datatype)); diff --git a/migrator/migrators/logs/migrations/000004_ttl_only_drop_parts.down.sql b/migrator/migrators/logs/migrations/000004_ttl_only_drop_parts.down.sql new file mode 100644 index 00000000..f02a9763 --- /dev/null +++ b/migrator/migrators/logs/migrations/000004_ttl_only_drop_parts.down.sql @@ -0,0 +1 @@ +ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} MODIFY SETTING ttl_only_drop_parts = 0; diff --git a/migrator/migrators/logs/migrations/000004_ttl_only_drop_parts.up.sql b/migrator/migrators/logs/migrations/000004_ttl_only_drop_parts.up.sql new file mode 100644 index 00000000..af88c596 --- /dev/null +++ b/migrator/migrators/logs/migrations/000004_ttl_only_drop_parts.up.sql @@ -0,0 +1 @@ +ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} MODIFY SETTING ttl_only_drop_parts = 1; diff --git a/exporter/clickhouselogsexporter/migrations/000005_attribute_rename.down.sql b/migrator/migrators/logs/migrations/000005_attribute_rename.down.sql similarity index 57% rename from exporter/clickhouselogsexporter/migrations/000005_attribute_rename.down.sql rename to migrator/migrators/logs/migrations/000005_attribute_rename.down.sql index e2c0ca2c..6219adf0 100644 --- a/exporter/clickhouselogsexporter/migrations/000005_attribute_rename.down.sql +++ b/migrator/migrators/logs/migrations/000005_attribute_rename.down.sql @@ -1,24 +1,24 @@ -RENAME TABLE IF EXISTS signoz_logs.logs_attribute_keys TO signoz_logs.logs_atrribute_keys on CLUSTER cluster; +RENAME TABLE IF EXISTS signoz_logs.logs_attribute_keys TO signoz_logs.logs_atrribute_keys ON CLUSTER {{.SIGNOZ_CLUSTER}}; -- Materialized view -DROP VIEW IF EXISTS attribute_keys_string_final_mv on CLUSTER cluster; -DROP VIEW IF EXISTS attribute_keys_int64_final_mv on CLUSTER cluster; -DROP VIEW IF EXISTS attribute_keys_float64_final_mv on CLUSTER cluster; +DROP VIEW IF EXISTS attribute_keys_string_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS attribute_keys_int64_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS attribute_keys_float64_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; -CREATE MATERIALIZED VIEW IF NOT EXISTS atrribute_keys_string_final_mv ON CLUSTER cluster TO signoz_logs.logs_atrribute_keys AS +CREATE MATERIALIZED VIEW IF NOT EXISTS atrribute_keys_string_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_logs.logs_atrribute_keys AS SELECT distinct arrayJoin(attributes_string_key) as name, 'String' datatype FROM signoz_logs.logs ORDER BY name; -CREATE MATERIALIZED VIEW IF NOT EXISTS atrribute_keys_int64_final_mv ON CLUSTER cluster TO signoz_logs.logs_atrribute_keys AS +CREATE MATERIALIZED VIEW IF NOT EXISTS atrribute_keys_int64_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_logs.logs_atrribute_keys AS SELECT distinct arrayJoin(attributes_int64_key) as name, 'Int64' datatype FROM signoz_logs.logs ORDER BY name; -CREATE MATERIALIZED VIEW IF NOT EXISTS atrribute_keys_float64_final_mv ON CLUSTER cluster TO signoz_logs.logs_atrribute_keys AS +CREATE MATERIALIZED VIEW IF NOT EXISTS atrribute_keys_float64_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_logs.logs_atrribute_keys AS SELECT distinct arrayJoin(attributes_float64_key) as name, 'Float64' datatype FROM signoz_logs.logs @@ -26,7 +26,7 @@ ORDER BY name; -- Distributed table -DROP TABLE IF EXISTS signoz_logs.distributed_logs_attribute_keys on CLUSTER cluster; +DROP TABLE IF EXISTS signoz_logs.distributed_logs_attribute_keys ON CLUSTER {{.SIGNOZ_CLUSTER}}; -CREATE TABLE IF NOT EXISTS signoz_logs.distributed_logs_atrribute_keys ON CLUSTER cluster AS signoz_logs.logs_atrribute_keys +CREATE TABLE IF NOT EXISTS signoz_logs.distributed_logs_atrribute_keys ON CLUSTER {{.SIGNOZ_CLUSTER}} AS signoz_logs.logs_atrribute_keys ENGINE = Distributed("cluster", "signoz_logs", logs_atrribute_keys, cityHash64(datatype)); \ No newline at end of file diff --git a/exporter/clickhouselogsexporter/migrations/000005_attribute_rename.up.sql b/migrator/migrators/logs/migrations/000005_attribute_rename.up.sql similarity index 57% rename from exporter/clickhouselogsexporter/migrations/000005_attribute_rename.up.sql rename to migrator/migrators/logs/migrations/000005_attribute_rename.up.sql index ff44581f..2846866d 100644 --- a/exporter/clickhouselogsexporter/migrations/000005_attribute_rename.up.sql +++ b/migrator/migrators/logs/migrations/000005_attribute_rename.up.sql @@ -1,24 +1,24 @@ -RENAME TABLE IF EXISTS signoz_logs.logs_atrribute_keys TO signoz_logs.logs_attribute_keys on CLUSTER cluster; +RENAME TABLE IF EXISTS signoz_logs.logs_atrribute_keys TO signoz_logs.logs_attribute_keys ON CLUSTER {{.SIGNOZ_CLUSTER}}; -- Materialized view -DROP VIEW IF EXISTS atrribute_keys_string_final_mv on CLUSTER cluster; -DROP VIEW IF EXISTS atrribute_keys_int64_final_mv on CLUSTER cluster; -DROP VIEW IF EXISTS atrribute_keys_float64_final_mv on CLUSTER cluster; +DROP VIEW IF EXISTS atrribute_keys_string_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS atrribute_keys_int64_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS atrribute_keys_float64_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; -CREATE MATERIALIZED VIEW IF NOT EXISTS attribute_keys_string_final_mv ON CLUSTER cluster TO signoz_logs.logs_attribute_keys AS +CREATE MATERIALIZED VIEW IF NOT EXISTS attribute_keys_string_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_logs.logs_attribute_keys AS SELECT distinct arrayJoin(attributes_string_key) as name, 'String' datatype FROM signoz_logs.logs ORDER BY name; -CREATE MATERIALIZED VIEW IF NOT EXISTS attribute_keys_int64_final_mv ON CLUSTER cluster TO signoz_logs.logs_attribute_keys AS +CREATE MATERIALIZED VIEW IF NOT EXISTS attribute_keys_int64_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_logs.logs_attribute_keys AS SELECT distinct arrayJoin(attributes_int64_key) as name, 'Int64' datatype FROM signoz_logs.logs ORDER BY name; -CREATE MATERIALIZED VIEW IF NOT EXISTS attribute_keys_float64_final_mv ON CLUSTER cluster TO signoz_logs.logs_attribute_keys AS +CREATE MATERIALIZED VIEW IF NOT EXISTS attribute_keys_float64_final_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_logs.logs_attribute_keys AS SELECT distinct arrayJoin(attributes_float64_key) as name, 'Float64' datatype FROM signoz_logs.logs @@ -26,7 +26,7 @@ ORDER BY name; -- Distributed table -DROP TABLE IF EXISTS signoz_logs.distributed_logs_atrribute_keys on CLUSTER cluster; +DROP TABLE IF EXISTS signoz_logs.distributed_logs_atrribute_keys ON CLUSTER {{.SIGNOZ_CLUSTER}}; -CREATE TABLE IF NOT EXISTS signoz_logs.distributed_logs_attribute_keys ON CLUSTER cluster AS signoz_logs.logs_attribute_keys +CREATE TABLE IF NOT EXISTS signoz_logs.distributed_logs_attribute_keys ON CLUSTER {{.SIGNOZ_CLUSTER}} AS signoz_logs.logs_attribute_keys ENGINE = Distributed("cluster", "signoz_logs", logs_attribute_keys, cityHash64(datatype)); \ No newline at end of file diff --git a/migrator/migrators/logs/migrations/000006_tag_attributes.down.sql b/migrator/migrators/logs/migrations/000006_tag_attributes.down.sql new file mode 100644 index 00000000..b7f6c13e --- /dev/null +++ b/migrator/migrators/logs/migrations/000006_tag_attributes.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS signoz_logs.distributed_tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP TABLE IF EXISTS signoz_logs.tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}}; \ No newline at end of file diff --git a/exporter/clickhouselogsexporter/migrations/000006_tag_attributes.up.sql b/migrator/migrators/logs/migrations/000006_tag_attributes.up.sql similarity index 90% rename from exporter/clickhouselogsexporter/migrations/000006_tag_attributes.up.sql rename to migrator/migrators/logs/migrations/000006_tag_attributes.up.sql index 8fd94c38..ae5a4864 100644 --- a/exporter/clickhouselogsexporter/migrations/000006_tag_attributes.up.sql +++ b/migrator/migrators/logs/migrations/000006_tag_attributes.up.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS signoz_logs.tag_attributes ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_logs.tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} ( timestamp DateTime CODEC(ZSTD(1)), tagKey LowCardinality(String) CODEC(ZSTD(1)), tagType Enum('tag', 'resource') CODEC(ZSTD(1)), @@ -11,5 +11,5 @@ ORDER BY (tagKey, tagType, tagDataType, stringTagValue, int64TagValue, float64Ta TTL toDateTime(timestamp) + INTERVAL 172800 SECOND DELETE SETTINGS ttl_only_drop_parts = 1, allow_nullable_key = 1; -CREATE TABLE IF NOT EXISTS signoz_logs.distributed_tag_attributes ON CLUSTER cluster AS signoz_logs.tag_attributes +CREATE TABLE IF NOT EXISTS signoz_logs.distributed_tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} AS signoz_logs.tag_attributes ENGINE = Distributed("cluster", "signoz_logs", tag_attributes, rand()); diff --git a/migrator/migrators/logs/migrations/000007_default_indexes.down.sql b/migrator/migrators/logs/migrations/000007_default_indexes.down.sql new file mode 100644 index 00000000..ab4b0367 --- /dev/null +++ b/migrator/migrators/logs/migrations/000007_default_indexes.down.sql @@ -0,0 +1,5 @@ +ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} drop index IF EXISTS severity_number_idx; + +ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} drop index IF EXISTS severity_text_idx; + +ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} drop index IF EXISTS trace_flags_idx; \ No newline at end of file diff --git a/exporter/clickhouselogsexporter/migrations/000007_default_indexes.up.sql b/migrator/migrators/logs/migrations/000007_default_indexes.up.sql similarity index 52% rename from exporter/clickhouselogsexporter/migrations/000007_default_indexes.up.sql rename to migrator/migrators/logs/migrations/000007_default_indexes.up.sql index b0e98cb0..1cd91e40 100644 --- a/exporter/clickhouselogsexporter/migrations/000007_default_indexes.up.sql +++ b/migrator/migrators/logs/migrations/000007_default_indexes.up.sql @@ -1,12 +1,12 @@ -- This migration adds default indexes to top level keys of the log model -- https://opentelemetry.io/docs/specs/otel/logs/data-model/#field-severitynumber range 1-24 and 0 means it's not set -ALTER TABLE signoz_logs.logs ON CLUSTER cluster add index IF NOT EXISTS severity_number_idx (severity_number) TYPE set(25) GRANULARITY 4; +ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} add index IF NOT EXISTS severity_number_idx (severity_number) TYPE set(25) GRANULARITY 4; -- https://opentelemetry.io/docs/specs/otel/logs/data-model/#displaying-severity 24 different values and empty means not set. -ALTER TABLE signoz_logs.logs ON CLUSTER cluster add index IF NOT EXISTS severity_text_idx (severity_text) TYPE set(25) GRANULARITY 4; +ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} add index IF NOT EXISTS severity_text_idx (severity_text) TYPE set(25) GRANULARITY 4; -- No point in addding index for trace_id, span_id as they are not set and they are always unique -- trace_flags can be a set so adding a default bloom filter -ALTER TABLE signoz_logs.logs ON CLUSTER cluster add index IF NOT EXISTS trace_flags_idx (trace_flags) TYPE bloom_filter() GRANULARITY 4; \ No newline at end of file +ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} add index IF NOT EXISTS trace_flags_idx (trace_flags) TYPE bloom_filter() GRANULARITY 4; \ No newline at end of file diff --git a/migrator/migrators/logs/migrator.go b/migrator/migrators/logs/migrator.go new file mode 100644 index 00000000..fb96c5bc --- /dev/null +++ b/migrator/migrators/logs/migrator.go @@ -0,0 +1,24 @@ +package logs + +import ( + "context" + basemigrator "github.com/SigNoz/signoz-otel-collector/migrator/migrators/baseMigrator" +) + +const ( + name = "logs" + database = "signoz_logs" + migrationFolder = "migrator/migrators/logs/migrations" +) + +type LogsMigrator struct { + *basemigrator.BaseMigrator +} + +func (m *LogsMigrator) Migrate(ctx context.Context) error { + return m.BaseMigrator.Migrate(ctx, database, migrationFolder) +} + +func (m *LogsMigrator) Name() string { + return name +} diff --git a/migrator/migrators/metrics/migrations/000001_init_db.down.sql b/migrator/migrators/metrics/migrations/000001_init_db.down.sql new file mode 100644 index 00000000..aa914d72 --- /dev/null +++ b/migrator/migrators/metrics/migrations/000001_init_db.down.sql @@ -0,0 +1,11 @@ +DROP TABLE IF EXISTS signoz_metrics.samples_v2 ON CLUSTER cluster; + +DROP TABLE IF EXISTS signoz_metrics.distributed_samples_v2 ON CLUSTER cluster; + +DROP TABLE IF EXISTS signoz_metrics.time_series_v2 ON CLUSTER cluster; + +DROP TABLE IF EXISTS signoz_metrics.distributed_time_series_v2 ON CLUSTER cluster; + +DROP TABLE IF EXISTS signoz_metrics.time_series_v3 ON CLUSTER cluster; + +DROP TABLE IF EXISTS signoz_metrics.distributed_time_series_v3 ON CLUSTER cluster; \ No newline at end of file diff --git a/migrator/migrators/metrics/migrations/000001_init_db.up.sql b/migrator/migrators/metrics/migrations/000001_init_db.up.sql new file mode 100644 index 00000000..b7d3d820 --- /dev/null +++ b/migrator/migrators/metrics/migrations/000001_init_db.up.sql @@ -0,0 +1,55 @@ +CREATE TABLE IF NOT EXISTS signoz_metrics.samples_v2 ON CLUSTER cluster ( + metric_name LowCardinality(String), + fingerprint UInt64 Codec(DoubleDelta, LZ4), + timestamp_ms Int64 Codec(DoubleDelta, LZ4), + value Float64 Codec(Gorilla, LZ4) + ) + ENGINE = MergeTree + PARTITION BY toDate(timestamp_ms / 1000) + ORDER BY (metric_name, fingerprint, timestamp_ms) + TTL toDateTime(timestamp_ms/1000) + INTERVAL 2592000 SECOND DELETE; + +CREATE TABLE IF NOT EXISTS signoz_metrics.distributed_samples_v2 ON CLUSTER cluster AS signoz_metrics.samples_v2 ENGINE = Distributed("cluster", "signoz_metrics", samples_v2, cityHash64(metric_name, fingerprint)); + +ALTER TABLE signoz_metrics.samples_v2 ON CLUSTER cluster MODIFY SETTING ttl_only_drop_parts = 1; + +SET allow_experimental_object_type = 1; + +CREATE TABLE IF NOT EXISTS signoz_metrics.time_series_v2 ON CLUSTER cluster( + metric_name LowCardinality(String), + fingerprint UInt64 Codec(DoubleDelta, LZ4), + timestamp_ms Int64 Codec(DoubleDelta, LZ4), + labels String Codec(ZSTD(5)) + ) + ENGINE = ReplacingMergeTree + PARTITION BY toDate(timestamp_ms / 1000) + ORDER BY (metric_name, fingerprint) + TTL toDateTime(timestamp_ms/1000) + INTERVAL 2592000 SECOND DELETE; + +CREATE TABLE IF NOT EXISTS signoz_metrics.distributed_time_series_v2 ON CLUSTER cluster AS signoz_metrics.time_series_v2 ENGINE = Distributed("cluster", signoz_metrics, time_series_v2, cityHash64(metric_name, fingerprint)); + +ALTER TABLE signoz_metrics.time_series_v2 ON CLUSTER cluster DROP COLUMN IF EXISTS labels_object; + +ALTER TABLE signoz_metrics.distributed_time_series_v2 ON CLUSTER cluster DROP COLUMN IF EXISTS labels_object; + +ALTER TABLE signoz_metrics.time_series_v2 ON CLUSTER cluster MODIFY SETTING ttl_only_drop_parts = 1; + +ALTER TABLE signoz_metrics.time_series_v2 ON CLUSTER cluster ADD COLUMN IF NOT EXISTS temporality LowCardinality(String) DEFAULT 'Unspecified' CODEC(ZSTD(5)); + +ALTER TABLE signoz_metrics.distributed_time_series_v2 ON CLUSTER cluster ADD COLUMN IF NOT EXISTS temporality LowCardinality(String) DEFAULT 'Unspecified' CODEC(ZSTD(5)); + +ALTER TABLE signoz_metrics.time_series_v2 ON CLUSTER cluster ADD INDEX IF NOT EXISTS temporality_index temporality TYPE SET(3) GRANULARITY 1; + +CREATE TABLE IF NOT EXISTS signoz_metrics.time_series_v3 ON CLUSTER cluster ( + env LowCardinality(String) DEFAULT 'default', + temporality LowCardinality(String) DEFAULT 'Unspecified', + metric_name LowCardinality(String), + fingerprint UInt64 CODEC(Delta, ZSTD), + timestamp_ms Int64 CODEC(Delta, ZSTD), + labels String CODEC(ZSTD(5)) + ) + ENGINE = ReplacingMergeTree + PARTITION BY toDate(timestamp_ms / 1000) + ORDER BY (env, temporality, metric_name, fingerprint); + +CREATE TABLE IF NOT EXISTS signoz_metrics.distributed_time_series_v3 ON CLUSTER cluster AS signoz_metrics.time_series_v3 ENGINE = Distributed("cluster", signoz_metrics, time_series_v3, cityHash64(env, temporality, metric_name, fingerprint)); \ No newline at end of file diff --git a/migrator/migrators/metrics/migrator.go b/migrator/migrators/metrics/migrator.go new file mode 100644 index 00000000..dbd20e12 --- /dev/null +++ b/migrator/migrators/metrics/migrator.go @@ -0,0 +1,40 @@ +package metrics + +import ( + "context" + "fmt" + "strings" + + "github.com/SigNoz/signoz-otel-collector/exporter/clickhousemetricsexporter" + basemigrator "github.com/SigNoz/signoz-otel-collector/migrator/migrators/baseMigrator" +) + +const ( + name = "metrics" + database = "signoz_metrics" + migrationFolder = "migrator/migrators/metrics/migrations" +) + +type MetricsMigrator struct { + *basemigrator.BaseMigrator +} + +func (m *MetricsMigrator) Migrate(ctx context.Context) error { + err := m.BaseMigrator.Migrate(ctx, database, migrationFolder) + if err != nil { + return err + } + + // TODO(srikanthccv): Remove this once we have a better way to handle data and last write + removeTTL := fmt.Sprintf(`ALTER TABLE %s.%s ON CLUSTER %s REMOVE TTL;`, database, clickhousemetricsexporter.TIME_SERIES_TABLE, m.Cfg.ClusterName) + if err = m.DB.Exec(context.Background(), removeTTL); err != nil { + if !strings.Contains(err.Error(), "Table doesn't have any table TTL expression, cannot remove.") { + return fmt.Errorf("failed to remove TTL from table %s.%s, err: %s", database, clickhousemetricsexporter.TIME_SERIES_TABLE, err) + } + } + return nil +} + +func (m *MetricsMigrator) Name() string { + return name +} diff --git a/migrator/migrators/migrators.go b/migrator/migrators/migrators.go new file mode 100644 index 00000000..315584ce --- /dev/null +++ b/migrator/migrators/migrators.go @@ -0,0 +1,18 @@ +package migrators + +import ( + "context" +) + +// MigratorConfig stores the configuration for a migrator +// Currently all migrators use the same config +type MigratorConfig struct { + DSN string + ClusterName string + IsMultiNodeCluster bool +} + +type Migrator interface { + Migrate(context.Context) error + Name() string +} diff --git a/migrator/migrators/traces/migrations/000001_init_db.down.sql b/migrator/migrators/traces/migrations/000001_init_db.down.sql new file mode 100644 index 00000000..72d5a503 --- /dev/null +++ b/migrator/migrators/traces/migrations/000001_init_db.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS signoz_traces.signoz_index ON CLUSTER {{.SIGNOZ_CLUSTER}}; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000001_init_db.up.sql b/migrator/migrators/traces/migrations/000001_init_db.up.sql similarity index 98% rename from exporter/clickhousetracesexporter/migrations/000001_init_db.up.sql rename to migrator/migrators/traces/migrations/000001_init_db.up.sql index 6d091eba..68e245ed 100644 --- a/exporter/clickhousetracesexporter/migrations/000001_init_db.up.sql +++ b/migrator/migrators/traces/migrations/000001_init_db.up.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS signoz_traces.signoz_index ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_traces.signoz_index ON CLUSTER {{.SIGNOZ_CLUSTER}} ( timestamp DateTime64(9) CODEC(Delta, ZSTD(1)), traceID String CODEC(ZSTD(1)), spanID String CODEC(ZSTD(1)), diff --git a/migrator/migrators/traces/migrations/000002_init_schema.down.sql b/migrator/migrators/traces/migrations/000002_init_schema.down.sql new file mode 100644 index 00000000..741b96e1 --- /dev/null +++ b/migrator/migrators/traces/migrations/000002_init_schema.down.sql @@ -0,0 +1 @@ +ALTER TABLE signoz_traces.signoz_index ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP COLUMN IF EXISTS events; \ No newline at end of file diff --git a/migrator/migrators/traces/migrations/000002_init_schema.up.sql b/migrator/migrators/traces/migrations/000002_init_schema.up.sql new file mode 100644 index 00000000..003105fc --- /dev/null +++ b/migrator/migrators/traces/migrations/000002_init_schema.up.sql @@ -0,0 +1 @@ +ALTER TABLE signoz_traces.signoz_index ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD COLUMN IF NOT EXISTS events Array(String); \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000003_init_schema.down.sql b/migrator/migrators/traces/migrations/000003_init_schema.down.sql similarity index 75% rename from exporter/clickhousetracesexporter/migrations/000003_init_schema.down.sql rename to migrator/migrators/traces/migrations/000003_init_schema.down.sql index 09451fb0..3d8b26c0 100644 --- a/exporter/clickhousetracesexporter/migrations/000003_init_schema.down.sql +++ b/migrator/migrators/traces/migrations/000003_init_schema.down.sql @@ -1 +1 @@ -DROP TABLE IF EXISTS signoz_traces.signoz_error_index ON CLUSTER cluster; \ No newline at end of file +DROP TABLE IF EXISTS signoz_traces.signoz_error_index ON CLUSTER {{.SIGNOZ_CLUSTER}}; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000003_init_schema.up.sql b/migrator/migrators/traces/migrations/000003_init_schema.up.sql similarity index 96% rename from exporter/clickhousetracesexporter/migrations/000003_init_schema.up.sql rename to migrator/migrators/traces/migrations/000003_init_schema.up.sql index 531c0edc..2209bdbc 100644 --- a/exporter/clickhousetracesexporter/migrations/000003_init_schema.up.sql +++ b/migrator/migrators/traces/migrations/000003_init_schema.up.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS signoz_traces.signoz_error_index ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_traces.signoz_error_index ON CLUSTER {{.SIGNOZ_CLUSTER}} ( timestamp DateTime64(9) CODEC(Delta, ZSTD(1)), errorID String CODEC(ZSTD(1)), traceID String CODEC(ZSTD(1)), diff --git a/migrator/migrators/traces/migrations/000004_update_table.down.sql b/migrator/migrators/traces/migrations/000004_update_table.down.sql new file mode 100644 index 00000000..9b506dcf --- /dev/null +++ b/migrator/migrators/traces/migrations/000004_update_table.down.sql @@ -0,0 +1 @@ +ALTER TABLE signoz_traces.signoz_index ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP COLUMN IF EXISTS httpMethod, DROP COLUMN IF EXISTS httpUrl, DROP COLUMN IF EXISTS httpCode, DROP COLUMN IF EXISTS httpRoute, DROP COLUMN IF EXISTS httpHost, DROP COLUMN IF EXISTS msgSystem, DROP COLUMN IF EXISTS msgOperation; \ No newline at end of file diff --git a/migrator/migrators/traces/migrations/000004_update_table.up.sql b/migrator/migrators/traces/migrations/000004_update_table.up.sql new file mode 100644 index 00000000..3f24f5f9 --- /dev/null +++ b/migrator/migrators/traces/migrations/000004_update_table.up.sql @@ -0,0 +1 @@ +ALTER TABLE signoz_traces.signoz_index ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD COLUMN IF NOT EXISTS httpMethod LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS httpUrl LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS httpCode LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS httpRoute LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS httpHost LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS msgSystem LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS msgOperation LowCardinality(String) CODEC(ZSTD(1)); diff --git a/migrator/migrators/traces/migrations/000005_add_hasError.down.sql b/migrator/migrators/traces/migrations/000005_add_hasError.down.sql new file mode 100644 index 00000000..a4611b8a --- /dev/null +++ b/migrator/migrators/traces/migrations/000005_add_hasError.down.sql @@ -0,0 +1 @@ +ALTER TABLE signoz_traces.signoz_index ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP COLUMN IF EXISTS hasError; \ No newline at end of file diff --git a/migrator/migrators/traces/migrations/000005_add_hasError.up.sql b/migrator/migrators/traces/migrations/000005_add_hasError.up.sql new file mode 100644 index 00000000..c342521f --- /dev/null +++ b/migrator/migrators/traces/migrations/000005_add_hasError.up.sql @@ -0,0 +1 @@ +ALTER TABLE signoz_traces.signoz_index ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD COLUMN IF NOT EXISTS hasError Int32; \ No newline at end of file diff --git a/migrator/migrators/traces/migrations/000006_add_tag_map.down.sql b/migrator/migrators/traces/migrations/000006_add_tag_map.down.sql new file mode 100644 index 00000000..6dad685f --- /dev/null +++ b/migrator/migrators/traces/migrations/000006_add_tag_map.down.sql @@ -0,0 +1 @@ +ALTER TABLE signoz_traces.signoz_index ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP COLUMN IF EXISTS tagMap; \ No newline at end of file diff --git a/migrator/migrators/traces/migrations/000006_add_tag_map.up.sql b/migrator/migrators/traces/migrations/000006_add_tag_map.up.sql new file mode 100644 index 00000000..c2458db9 --- /dev/null +++ b/migrator/migrators/traces/migrations/000006_add_tag_map.up.sql @@ -0,0 +1 @@ +ALTER TABLE signoz_traces.signoz_index ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD COLUMN IF NOT EXISTS tagMap Map(LowCardinality(String), String); \ No newline at end of file diff --git a/migrator/migrators/traces/migrations/000007_init_signoz_index_v2.down.sql b/migrator/migrators/traces/migrations/000007_init_signoz_index_v2.down.sql new file mode 100644 index 00000000..274ae836 --- /dev/null +++ b/migrator/migrators/traces/migrations/000007_init_signoz_index_v2.down.sql @@ -0,0 +1,3 @@ +DROP TABLE IF EXISTS signoz_traces.signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}}; + +SET allow_experimental_projection_optimization = 0; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000007_init_signoz_index_v2.up.sql b/migrator/migrators/traces/migrations/000007_init_signoz_index_v2.up.sql similarity index 98% rename from exporter/clickhousetracesexporter/migrations/000007_init_signoz_index_v2.up.sql rename to migrator/migrators/traces/migrations/000007_init_signoz_index_v2.up.sql index a88830d2..c77bd569 100644 --- a/exporter/clickhousetracesexporter/migrations/000007_init_signoz_index_v2.up.sql +++ b/migrator/migrators/traces/migrations/000007_init_signoz_index_v2.up.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS signoz_traces.signoz_index_v2 ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_traces.signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} ( timestamp DateTime64(9) CODEC(DoubleDelta, LZ4), traceID FixedString(32) CODEC(ZSTD(1)), spanID String CODEC(ZSTD(1)), diff --git a/migrator/migrators/traces/migrations/000008_init_signoz_spans.down.sql b/migrator/migrators/traces/migrations/000008_init_signoz_spans.down.sql new file mode 100644 index 00000000..721faf06 --- /dev/null +++ b/migrator/migrators/traces/migrations/000008_init_signoz_spans.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS signoz_traces.signoz_spans ON CLUSTER {{.SIGNOZ_CLUSTER}}; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000008_init_signoz_spans.up.sql b/migrator/migrators/traces/migrations/000008_init_signoz_spans.up.sql similarity index 93% rename from exporter/clickhousetracesexporter/migrations/000008_init_signoz_spans.up.sql rename to migrator/migrators/traces/migrations/000008_init_signoz_spans.up.sql index 3215f566..10af9a6f 100644 --- a/exporter/clickhousetracesexporter/migrations/000008_init_signoz_spans.up.sql +++ b/migrator/migrators/traces/migrations/000008_init_signoz_spans.up.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS signoz_traces.signoz_spans ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_traces.signoz_spans ON CLUSTER {{.SIGNOZ_CLUSTER}} ( timestamp DateTime64(9) CODEC(DoubleDelta, LZ4), traceID FixedString(32) CODEC(ZSTD(1)), model String CODEC(ZSTD(9)) diff --git a/exporter/clickhousetracesexporter/migrations/000009_init_signoz_error_index.down.sql b/migrator/migrators/traces/migrations/000009_init_signoz_error_index.down.sql similarity index 75% rename from exporter/clickhousetracesexporter/migrations/000009_init_signoz_error_index.down.sql rename to migrator/migrators/traces/migrations/000009_init_signoz_error_index.down.sql index 09451fb0..3d8b26c0 100644 --- a/exporter/clickhousetracesexporter/migrations/000009_init_signoz_error_index.down.sql +++ b/migrator/migrators/traces/migrations/000009_init_signoz_error_index.down.sql @@ -1 +1 @@ -DROP TABLE IF EXISTS signoz_traces.signoz_error_index ON CLUSTER cluster; \ No newline at end of file +DROP TABLE IF EXISTS signoz_traces.signoz_error_index ON CLUSTER {{.SIGNOZ_CLUSTER}}; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000009_init_signoz_error_index.up.sql b/migrator/migrators/traces/migrations/000009_init_signoz_error_index.up.sql similarity index 96% rename from exporter/clickhousetracesexporter/migrations/000009_init_signoz_error_index.up.sql rename to migrator/migrators/traces/migrations/000009_init_signoz_error_index.up.sql index 2ca83caf..ca245b2d 100644 --- a/exporter/clickhousetracesexporter/migrations/000009_init_signoz_error_index.up.sql +++ b/migrator/migrators/traces/migrations/000009_init_signoz_error_index.up.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS signoz_traces.signoz_error_index ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_traces.signoz_error_index ON CLUSTER {{.SIGNOZ_CLUSTER}} ( timestamp DateTime64(9) CODEC(DoubleDelta, LZ4), errorID FixedString(32) CODEC(ZSTD(1)), traceID FixedString(32) CODEC(ZSTD(1)), diff --git a/migrator/migrators/traces/migrations/000010_add_gRPC_code_method.down.sql b/migrator/migrators/traces/migrations/000010_add_gRPC_code_method.down.sql new file mode 100644 index 00000000..7a69239d --- /dev/null +++ b/migrator/migrators/traces/migrations/000010_add_gRPC_code_method.down.sql @@ -0,0 +1 @@ +ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP COLUMN IF EXISTS gRPCCode, DROP COLUMN IF EXISTS gRPCMethod; \ No newline at end of file diff --git a/migrator/migrators/traces/migrations/000010_add_gRPC_code_method.up.sql b/migrator/migrators/traces/migrations/000010_add_gRPC_code_method.up.sql new file mode 100644 index 00000000..625491b7 --- /dev/null +++ b/migrator/migrators/traces/migrations/000010_add_gRPC_code_method.up.sql @@ -0,0 +1 @@ +ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD COLUMN IF NOT EXISTS gRPCMethod LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS gRPCCode LowCardinality(String) CODEC(ZSTD(1)); \ No newline at end of file diff --git a/migrator/migrators/traces/migrations/000011_init_durationMV.down.sql b/migrator/migrators/traces/migrations/000011_init_durationMV.down.sql new file mode 100644 index 00000000..e1651e09 --- /dev/null +++ b/migrator/migrators/traces/migrations/000011_init_durationMV.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS signoz_traces.durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.durationSortMV ON CLUSTER {{.SIGNOZ_CLUSTER}}; diff --git a/exporter/clickhousetracesexporter/migrations/000011_init_durationMV.up.sql b/migrator/migrators/traces/migrations/000011_init_durationMV.up.sql similarity index 97% rename from exporter/clickhousetracesexporter/migrations/000011_init_durationMV.up.sql rename to migrator/migrators/traces/migrations/000011_init_durationMV.up.sql index ebc76819..214ff778 100644 --- a/exporter/clickhousetracesexporter/migrations/000011_init_durationMV.up.sql +++ b/migrator/migrators/traces/migrations/000011_init_durationMV.up.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS signoz_traces.durationSort ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_traces.durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} ( timestamp DateTime64(9) CODEC(DoubleDelta, LZ4), traceID FixedString(32) CODEC(ZSTD(1)), spanID String CODEC(ZSTD(1)), @@ -37,7 +37,7 @@ ORDER BY (durationNano, timestamp) TTL toDateTime(timestamp) + INTERVAL 604800 SECOND DELETE SETTINGS index_granularity = 8192; -CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.durationSortMV ON CLUSTER cluster +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.durationSortMV ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_traces.durationSort AS SELECT timestamp, diff --git a/exporter/clickhousetracesexporter/migrations/000012_init_signoz_error_index_v2.down.sql b/migrator/migrators/traces/migrations/000012_init_signoz_error_index_v2.down.sql similarity index 72% rename from exporter/clickhousetracesexporter/migrations/000012_init_signoz_error_index_v2.down.sql rename to migrator/migrators/traces/migrations/000012_init_signoz_error_index_v2.down.sql index 957df960..aab37a63 100644 --- a/exporter/clickhousetracesexporter/migrations/000012_init_signoz_error_index_v2.down.sql +++ b/migrator/migrators/traces/migrations/000012_init_signoz_error_index_v2.down.sql @@ -1 +1 @@ -DROP TABLE IF EXISTS signoz_traces.signoz_error_index_v2 ON CLUSTER cluster; \ No newline at end of file +DROP TABLE IF EXISTS signoz_traces.signoz_error_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}}; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000012_init_signoz_error_index_v2.up.sql b/migrator/migrators/traces/migrations/000012_init_signoz_error_index_v2.up.sql similarity index 95% rename from exporter/clickhousetracesexporter/migrations/000012_init_signoz_error_index_v2.up.sql rename to migrator/migrators/traces/migrations/000012_init_signoz_error_index_v2.up.sql index a71a183b..99fa5bd7 100644 --- a/exporter/clickhousetracesexporter/migrations/000012_init_signoz_error_index_v2.up.sql +++ b/migrator/migrators/traces/migrations/000012_init_signoz_error_index_v2.up.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS signoz_traces.signoz_error_index_v2 ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_traces.signoz_error_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} ( timestamp DateTime64(9) CODEC(DoubleDelta, LZ4), errorID FixedString(32) CODEC(ZSTD(1)), groupID FixedString(32) CODEC(ZSTD(1)), diff --git a/exporter/clickhousetracesexporter/migrations/000013_add_RPC_system_service_method.down.sql b/migrator/migrators/traces/migrations/000013_add_RPC_system_service_method.down.sql similarity index 68% rename from exporter/clickhousetracesexporter/migrations/000013_add_RPC_system_service_method.down.sql rename to migrator/migrators/traces/migrations/000013_add_RPC_system_service_method.down.sql index e2d5e633..7cae95f2 100644 --- a/exporter/clickhousetracesexporter/migrations/000013_add_RPC_system_service_method.down.sql +++ b/migrator/migrators/traces/migrations/000013_add_RPC_system_service_method.down.sql @@ -1,17 +1,17 @@ -DROP TABLE IF EXISTS signoz_traces.durationSortMV ON CLUSTER cluster; +DROP TABLE IF EXISTS signoz_traces.durationSortMV ON CLUSTER {{.SIGNOZ_CLUSTER}}; -ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER cluster +ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP INDEX idx_rpcMethod, DROP INDEX idx_responseStatusCode; -ALTER TABLE signoz_traces.durationSort ON CLUSTER cluster +ALTER TABLE signoz_traces.durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP INDEX idx_rpcMethod, DROP INDEX idx_responseStatusCode; -CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.durationSortMV ON CLUSTER cluster +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.durationSortMV ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_traces.durationSort AS SELECT timestamp, @@ -36,4 +36,4 @@ AS SELECT FROM signoz_traces.signoz_index_v2 ORDER BY durationNano, timestamp; -ATTACH TABLE IF NOT EXISTS signoz_traces.durationSortMV ON CLUSTER cluster; \ No newline at end of file +ATTACH TABLE IF NOT EXISTS signoz_traces.durationSortMV ON CLUSTER {{.SIGNOZ_CLUSTER}}; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000013_add_RPC_system_service_method.up.sql b/migrator/migrators/traces/migrations/000013_add_RPC_system_service_method.up.sql similarity index 78% rename from exporter/clickhousetracesexporter/migrations/000013_add_RPC_system_service_method.up.sql rename to migrator/migrators/traces/migrations/000013_add_RPC_system_service_method.up.sql index 0158f6b1..ef37757d 100644 --- a/exporter/clickhousetracesexporter/migrations/000013_add_RPC_system_service_method.up.sql +++ b/migrator/migrators/traces/migrations/000013_add_RPC_system_service_method.up.sql @@ -1,6 +1,6 @@ -DROP TABLE IF EXISTS signoz_traces.durationSortMV ON CLUSTER cluster; +DROP TABLE IF EXISTS signoz_traces.durationSortMV ON CLUSTER {{.SIGNOZ_CLUSTER}}; -ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER cluster +ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD COLUMN IF NOT EXISTS `rpcSystem` LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS `rpcService` LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS `rpcMethod` LowCardinality(String) CODEC(ZSTD(1)), @@ -8,7 +8,7 @@ ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER cluster -ALTER TABLE signoz_traces.durationSort ON CLUSTER cluster +ALTER TABLE signoz_traces.durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD COLUMN IF NOT EXISTS `rpcSystem` LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS `rpcService` LowCardinality(String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS `rpcMethod` LowCardinality(String) CODEC(ZSTD(1)), @@ -16,17 +16,17 @@ ALTER TABLE signoz_traces.durationSort ON CLUSTER cluster -ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER cluster +ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD INDEX IF NOT EXISTS idx_rpcMethod rpcMethod TYPE bloom_filter GRANULARITY 4, ADD INDEX IF NOT EXISTS idx_responseStatusCode responseStatusCode TYPE set(0) GRANULARITY 1; -ALTER TABLE signoz_traces.durationSort ON CLUSTER cluster +ALTER TABLE signoz_traces.durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD INDEX IF NOT EXISTS idx_rpcMethod rpcMethod TYPE bloom_filter GRANULARITY 4, ADD INDEX IF NOT EXISTS idx_responseStatusCode responseStatusCode TYPE set(0) GRANULARITY 1; -CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.durationSortMV ON CLUSTER cluster +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.durationSortMV ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_traces.durationSort AS SELECT timestamp, diff --git a/migrator/migrators/traces/migrations/000014_add_usage_explorer_mv.down.sql b/migrator/migrators/traces/migrations/000014_add_usage_explorer_mv.down.sql new file mode 100644 index 00000000..e2967ae8 --- /dev/null +++ b/migrator/migrators/traces/migrations/000014_add_usage_explorer_mv.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS signoz_traces.usage_explorer ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.usage_explorer_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; diff --git a/exporter/clickhousetracesexporter/migrations/000014_add_usage_explorer_mv.up.sql b/migrator/migrators/traces/migrations/000014_add_usage_explorer_mv.up.sql similarity index 90% rename from exporter/clickhousetracesexporter/migrations/000014_add_usage_explorer_mv.up.sql rename to migrator/migrators/traces/migrations/000014_add_usage_explorer_mv.up.sql index 7fdc7505..8e09856f 100644 --- a/exporter/clickhousetracesexporter/migrations/000014_add_usage_explorer_mv.up.sql +++ b/migrator/migrators/traces/migrations/000014_add_usage_explorer_mv.up.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS signoz_traces.usage_explorer ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_traces.usage_explorer ON CLUSTER {{.SIGNOZ_CLUSTER}} ( timestamp DateTime64(9) CODEC(DoubleDelta, LZ4), service_name LowCardinality(String) CODEC(ZSTD(1)), count UInt64 CODEC(T64, ZSTD(1)) @@ -9,7 +9,7 @@ TTL toDateTime(timestamp) + INTERVAL 604800 SECOND DELETE; -CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.usage_explorer_mv ON CLUSTER cluster +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.usage_explorer_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_traces.usage_explorer AS SELECT toStartOfHour(timestamp) as timestamp, diff --git a/exporter/clickhousetracesexporter/migrations/000015_top_level_operations_mv.down.sql b/migrator/migrators/traces/migrations/000015_top_level_operations_mv.down.sql similarity index 50% rename from exporter/clickhousetracesexporter/migrations/000015_top_level_operations_mv.down.sql rename to migrator/migrators/traces/migrations/000015_top_level_operations_mv.down.sql index 0559d365..6a244c2c 100644 --- a/exporter/clickhousetracesexporter/migrations/000015_top_level_operations_mv.down.sql +++ b/migrator/migrators/traces/migrations/000015_top_level_operations_mv.down.sql @@ -1,3 +1,3 @@ -DROP TABLE IF EXISTS signoz_traces.top_level_operations ON CLUSTER cluster; -DROP VIEW IF EXISTS signoz_traces.sub_root_operations ON CLUSTER cluster; -DROP VIEW IF EXISTS signoz_traces.root_operations ON CLUSTER cluster; +DROP TABLE IF EXISTS signoz_traces.top_level_operations ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.sub_root_operations ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.root_operations ON CLUSTER {{.SIGNOZ_CLUSTER}}; diff --git a/exporter/clickhousetracesexporter/migrations/000015_top_level_operations_mv.up.sql b/migrator/migrators/traces/migrations/000015_top_level_operations_mv.up.sql similarity index 86% rename from exporter/clickhousetracesexporter/migrations/000015_top_level_operations_mv.up.sql rename to migrator/migrators/traces/migrations/000015_top_level_operations_mv.up.sql index 6a9c9afe..e06fdb21 100644 --- a/exporter/clickhousetracesexporter/migrations/000015_top_level_operations_mv.up.sql +++ b/migrator/migrators/traces/migrations/000015_top_level_operations_mv.up.sql @@ -1,11 +1,11 @@ -CREATE TABLE IF NOT EXISTS signoz_traces.top_level_operations ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_traces.top_level_operations ON CLUSTER {{.SIGNOZ_CLUSTER}} ( name LowCardinality(String) CODEC(ZSTD(1)), serviceName LowCardinality(String) CODEC(ZSTD(1)) ) ENGINE ReplacingMergeTree ORDER BY (serviceName, name); -CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.sub_root_operations ON CLUSTER cluster +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.sub_root_operations ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_traces.top_level_operations AS SELECT DISTINCT name, @@ -13,7 +13,7 @@ AS SELECT DISTINCT FROM signoz_traces.signoz_index_v2 AS A, signoz_traces.signoz_index_v2 AS B WHERE (A.serviceName != B.serviceName) AND (A.parentSpanID = B.spanID); -CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.root_operations ON CLUSTER cluster +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.root_operations ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_traces.top_level_operations AS SELECT DISTINCT name, diff --git a/exporter/clickhousetracesexporter/migrations/000016_dependency_graph_minutes.down.sql b/migrator/migrators/traces/migrations/000016_dependency_graph_minutes.down.sql similarity index 61% rename from exporter/clickhousetracesexporter/migrations/000016_dependency_graph_minutes.down.sql rename to migrator/migrators/traces/migrations/000016_dependency_graph_minutes.down.sql index b18a73d6..a5573c1d 100644 --- a/exporter/clickhousetracesexporter/migrations/000016_dependency_graph_minutes.down.sql +++ b/migrator/migrators/traces/migrations/000016_dependency_graph_minutes.down.sql @@ -1,4 +1,4 @@ -DROP TABLE IF EXISTS signoz_traces.dependency_graph_minutes ON CLUSTER cluster; -DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_service_calls_mv ON CLUSTER cluster; -DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_db_calls_mv ON CLUSTER cluster; -DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_messaging_calls_mv ON CLUSTER cluster; +DROP TABLE IF EXISTS signoz_traces.dependency_graph_minutes ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_service_calls_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_db_calls_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_messaging_calls_mv ON CLUSTER {{.SIGNOZ_CLUSTER}}; diff --git a/exporter/clickhousetracesexporter/migrations/000016_dependency_graph_minutes.up.sql b/migrator/migrators/traces/migrations/000016_dependency_graph_minutes.up.sql similarity index 90% rename from exporter/clickhousetracesexporter/migrations/000016_dependency_graph_minutes.up.sql rename to migrator/migrators/traces/migrations/000016_dependency_graph_minutes.up.sql index 0b275854..1b15365f 100644 --- a/exporter/clickhousetracesexporter/migrations/000016_dependency_graph_minutes.up.sql +++ b/migrator/migrators/traces/migrations/000016_dependency_graph_minutes.up.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS signoz_traces.dependency_graph_minutes ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_traces.dependency_graph_minutes ON CLUSTER {{.SIGNOZ_CLUSTER}} ( src LowCardinality(String) CODEC(ZSTD(1)), dest LowCardinality(String) CODEC(ZSTD(1)), duration_quantiles_state AggregateFunction(quantiles(0.5, 0.75, 0.9, 0.95, 0.99), Float64) CODEC(Default), @@ -12,7 +12,7 @@ TTL toDateTime(timestamp) + INTERVAL 604800 SECOND DELETE; -CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_service_calls_mv ON CLUSTER cluster +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_service_calls_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_traces.dependency_graph_minutes AS SELECT A.serviceName as src, @@ -25,7 +25,7 @@ FROM signoz_traces.signoz_index_v2 AS A, signoz_traces.signoz_index_v2 AS B WHERE (A.serviceName != B.serviceName) AND (A.spanID = B.parentSpanID) GROUP BY timestamp, src, dest; -CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_db_calls_mv ON CLUSTER cluster +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_db_calls_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_traces.dependency_graph_minutes AS SELECT serviceName as src, @@ -38,7 +38,7 @@ FROM signoz_traces.signoz_index_v2 WHERE dest != '' and kind != 2 GROUP BY timestamp, src, dest; -CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_messaging_calls_mv ON CLUSTER cluster +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_messaging_calls_mv ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_traces.dependency_graph_minutes AS SELECT serviceName as src, diff --git a/exporter/clickhousetracesexporter/migrations/000017_add_distributed_table.down.sql b/migrator/migrators/traces/migrations/000017_add_distributed_table.down.sql similarity index 66% rename from exporter/clickhousetracesexporter/migrations/000017_add_distributed_table.down.sql rename to migrator/migrators/traces/migrations/000017_add_distributed_table.down.sql index e22f1d4a..93d93cdd 100644 --- a/exporter/clickhousetracesexporter/migrations/000017_add_distributed_table.down.sql +++ b/migrator/migrators/traces/migrations/000017_add_distributed_table.down.sql @@ -1,13 +1,13 @@ -DROP TABLE IF EXISTS signoz_traces.distributed_signoz_index_v2 ON CLUSTER cluster; +DROP TABLE IF EXISTS signoz_traces.distributed_signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}}; -DROP TABLE IF EXISTS signoz_traces.distributed_signoz_spans ON CLUSTER cluster; +DROP TABLE IF EXISTS signoz_traces.distributed_signoz_spans ON CLUSTER {{.SIGNOZ_CLUSTER}}; -DROP TABLE IF EXISTS signoz_traces.distributed_durationSort ON CLUSTER cluster; +DROP TABLE IF EXISTS signoz_traces.distributed_durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}}; -DROP TABLE IF EXISTS signoz_traces.distributed_signoz_error_index_v2 ON CLUSTER cluster; +DROP TABLE IF EXISTS signoz_traces.distributed_signoz_error_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}}; -DROP TABLE IF EXISTS signoz_traces.distributed_usage_explorer ON CLUSTER cluster; +DROP TABLE IF EXISTS signoz_traces.distributed_usage_explorer ON CLUSTER {{.SIGNOZ_CLUSTER}}; -DROP TABLE IF EXISTS signoz_traces.distributed_top_level_operations ON CLUSTER cluster; +DROP TABLE IF EXISTS signoz_traces.distributed_top_level_operations ON CLUSTER {{.SIGNOZ_CLUSTER}}; -DROP TABLE IF EXISTS signoz_traces.distributed_dependency_graph_minutes ON CLUSTER cluster \ No newline at end of file +DROP TABLE IF EXISTS signoz_traces.distributed_dependency_graph_minutes ON CLUSTER {{.SIGNOZ_CLUSTER}} \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000017_add_distributed_table.up.sql b/migrator/migrators/traces/migrations/000017_add_distributed_table.up.sql similarity index 68% rename from exporter/clickhousetracesexporter/migrations/000017_add_distributed_table.up.sql rename to migrator/migrators/traces/migrations/000017_add_distributed_table.up.sql index eba3a10b..ba6f8841 100644 --- a/exporter/clickhousetracesexporter/migrations/000017_add_distributed_table.up.sql +++ b/migrator/migrators/traces/migrations/000017_add_distributed_table.up.sql @@ -1,21 +1,21 @@ -CREATE TABLE IF NOT EXISTS signoz_traces.distributed_signoz_index_v2 ON CLUSTER cluster AS signoz_traces.signoz_index_v2 +CREATE TABLE IF NOT EXISTS signoz_traces.distributed_signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} AS signoz_traces.signoz_index_v2 ENGINE = Distributed("cluster", "signoz_traces", signoz_index_v2, cityHash64(traceID)); -CREATE TABLE IF NOT EXISTS signoz_traces.distributed_signoz_spans ON CLUSTER cluster AS signoz_traces.signoz_spans +CREATE TABLE IF NOT EXISTS signoz_traces.distributed_signoz_spans ON CLUSTER {{.SIGNOZ_CLUSTER}} AS signoz_traces.signoz_spans ENGINE = Distributed("cluster", "signoz_traces", signoz_spans, cityHash64(traceID)); -CREATE TABLE IF NOT EXISTS signoz_traces.distributed_durationSort ON CLUSTER cluster AS signoz_traces.durationSort +CREATE TABLE IF NOT EXISTS signoz_traces.distributed_durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} AS signoz_traces.durationSort ENGINE = Distributed("cluster", "signoz_traces", durationSort, cityHash64(traceID)); -CREATE TABLE IF NOT EXISTS signoz_traces.distributed_signoz_error_index_v2 ON CLUSTER cluster AS signoz_traces.signoz_error_index_v2 +CREATE TABLE IF NOT EXISTS signoz_traces.distributed_signoz_error_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} AS signoz_traces.signoz_error_index_v2 ENGINE = Distributed("cluster", "signoz_traces", signoz_error_index_v2, cityHash64(groupID)); -CREATE TABLE IF NOT EXISTS signoz_traces.distributed_usage_explorer ON CLUSTER cluster AS signoz_traces.usage_explorer +CREATE TABLE IF NOT EXISTS signoz_traces.distributed_usage_explorer ON CLUSTER {{.SIGNOZ_CLUSTER}} AS signoz_traces.usage_explorer ENGINE = Distributed("cluster", "signoz_traces", usage_explorer, cityHash64(rand())); -CREATE TABLE IF NOT EXISTS signoz_traces.distributed_top_level_operations ON CLUSTER cluster AS signoz_traces.top_level_operations +CREATE TABLE IF NOT EXISTS signoz_traces.distributed_top_level_operations ON CLUSTER {{.SIGNOZ_CLUSTER}} AS signoz_traces.top_level_operations ENGINE = Distributed("cluster", "signoz_traces", top_level_operations, cityHash64(rand())); -CREATE TABLE IF NOT EXISTS signoz_traces.distributed_dependency_graph_minutes ON CLUSTER cluster AS signoz_traces.dependency_graph_minutes +CREATE TABLE IF NOT EXISTS signoz_traces.distributed_dependency_graph_minutes ON CLUSTER {{.SIGNOZ_CLUSTER}} AS signoz_traces.dependency_graph_minutes ENGINE = Distributed("cluster", "signoz_traces", dependency_graph_minutes, cityHash64(rand())); diff --git a/exporter/clickhousetracesexporter/migrations/000018_add_new_tag_types.down.sql b/migrator/migrators/traces/migrations/000018_add_new_tag_types.down.sql similarity index 79% rename from exporter/clickhousetracesexporter/migrations/000018_add_new_tag_types.down.sql rename to migrator/migrators/traces/migrations/000018_add_new_tag_types.down.sql index 7386b0b9..bcf27386 100644 --- a/exporter/clickhousetracesexporter/migrations/000018_add_new_tag_types.down.sql +++ b/migrator/migrators/traces/migrations/000018_add_new_tag_types.down.sql @@ -1,26 +1,26 @@ -DROP TABLE IF EXISTS signoz_traces.durationSortMV ON CLUSTER cluster; +DROP TABLE IF EXISTS signoz_traces.durationSortMV ON CLUSTER {{.SIGNOZ_CLUSTER}}; -ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER cluster +ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP COLUMN IF NOT EXISTS stringTagMap Map(String, String) CODEC(ZSTD(1)), DROP COLUMN IF NOT EXISTS numberTagMap Map(String, Float64) CODEC(ZSTD(1)), DROP COLUMN IF NOT EXISTS boolTagMap Map(String, bool) CODEC(ZSTD(1)); -ALTER TABLE signoz_traces.distributed_signoz_index_v2 ON CLUSTER cluster +ALTER TABLE signoz_traces.distributed_signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP COLUMN IF NOT EXISTS stringTagMap Map(String, String) CODEC(ZSTD(1)), DROP COLUMN IF NOT EXISTS numberTagMap Map(String, Float64) CODEC(ZSTD(1)), DROP COLUMN IF NOT EXISTS boolTagMap Map(String, bool) CODEC(ZSTD(1)); -ALTER TABLE signoz_traces.durationSort ON CLUSTER cluster +ALTER TABLE signoz_traces.durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP COLUMN IF NOT EXISTS stringTagMap Map(String, String) CODEC(ZSTD(1)), DROP COLUMN IF NOT EXISTS numberTagMap Map(String, Float64) CODEC(ZSTD(1)), DROP COLUMN IF NOT EXISTS boolTagMap Map(String, bool) CODEC(ZSTD(1)); -ALTER TABLE signoz_traces.distributed_durationSort ON CLUSTER cluster +ALTER TABLE signoz_traces.distributed_durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP COLUMN IF NOT EXISTS stringTagMap Map(String, String) CODEC(ZSTD(1)), DROP COLUMN IF NOT EXISTS numberTagMap Map(String, Float64) CODEC(ZSTD(1)), DROP COLUMN IF NOT EXISTS boolTagMap Map(String, bool) CODEC(ZSTD(1)); -CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.durationSortMV ON CLUSTER cluster +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.durationSortMV ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_traces.durationSort AS SELECT timestamp, diff --git a/exporter/clickhousetracesexporter/migrations/000018_add_new_tag_types.up.sql b/migrator/migrators/traces/migrations/000018_add_new_tag_types.up.sql similarity index 79% rename from exporter/clickhousetracesexporter/migrations/000018_add_new_tag_types.up.sql rename to migrator/migrators/traces/migrations/000018_add_new_tag_types.up.sql index fefe195f..4a7241c1 100644 --- a/exporter/clickhousetracesexporter/migrations/000018_add_new_tag_types.up.sql +++ b/migrator/migrators/traces/migrations/000018_add_new_tag_types.up.sql @@ -1,26 +1,26 @@ -DROP TABLE IF EXISTS signoz_traces.durationSortMV ON CLUSTER cluster; +DROP TABLE IF EXISTS signoz_traces.durationSortMV ON CLUSTER {{.SIGNOZ_CLUSTER}}; -ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER cluster +ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD COLUMN IF NOT EXISTS stringTagMap Map(String, String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS numberTagMap Map(String, Float64) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS boolTagMap Map(String, bool) CODEC(ZSTD(1)); -ALTER TABLE signoz_traces.distributed_signoz_index_v2 ON CLUSTER cluster +ALTER TABLE signoz_traces.distributed_signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD COLUMN IF NOT EXISTS stringTagMap Map(String, String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS numberTagMap Map(String, Float64) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS boolTagMap Map(String, bool) CODEC(ZSTD(1)); -ALTER TABLE signoz_traces.durationSort ON CLUSTER cluster +ALTER TABLE signoz_traces.durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD COLUMN IF NOT EXISTS stringTagMap Map(String, String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS numberTagMap Map(String, Float64) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS boolTagMap Map(String, bool) CODEC(ZSTD(1)); -ALTER TABLE signoz_traces.distributed_durationSort ON CLUSTER cluster +ALTER TABLE signoz_traces.distributed_durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD COLUMN IF NOT EXISTS stringTagMap Map(String, String) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS numberTagMap Map(String, Float64) CODEC(ZSTD(1)), ADD COLUMN IF NOT EXISTS boolTagMap Map(String, bool) CODEC(ZSTD(1)); -CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.durationSortMV ON CLUSTER cluster +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.durationSortMV ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_traces.durationSort AS SELECT timestamp, diff --git a/migrator/migrators/traces/migrations/000019_ttl_only_drop_parts.down.sql b/migrator/migrators/traces/migrations/000019_ttl_only_drop_parts.down.sql new file mode 100644 index 00000000..c0f5c2ae --- /dev/null +++ b/migrator/migrators/traces/migrations/000019_ttl_only_drop_parts.down.sql @@ -0,0 +1,6 @@ +ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} MODIFY SETTING ttl_only_drop_parts = 0; +ALTER TABLE signoz_traces.signoz_error_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} MODIFY SETTING ttl_only_drop_parts = 0; +ALTER TABLE signoz_traces.signoz_spans ON CLUSTER {{.SIGNOZ_CLUSTER}} MODIFY SETTING ttl_only_drop_parts = 0; +ALTER TABLE signoz_traces.durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} MODIFY SETTING ttl_only_drop_parts = 0; +ALTER TABLE signoz_traces.dependency_graph_minutes ON CLUSTER {{.SIGNOZ_CLUSTER}} MODIFY SETTING ttl_only_drop_parts = 0; +ALTER TABLE signoz_traces.usage_explorer ON CLUSTER {{.SIGNOZ_CLUSTER}} MODIFY SETTING ttl_only_drop_parts = 0; \ No newline at end of file diff --git a/migrator/migrators/traces/migrations/000019_ttl_only_drop_parts.up.sql b/migrator/migrators/traces/migrations/000019_ttl_only_drop_parts.up.sql new file mode 100644 index 00000000..971dc58f --- /dev/null +++ b/migrator/migrators/traces/migrations/000019_ttl_only_drop_parts.up.sql @@ -0,0 +1,6 @@ +ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} MODIFY SETTING ttl_only_drop_parts = 1; +ALTER TABLE signoz_traces.signoz_error_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} MODIFY SETTING ttl_only_drop_parts = 1; +ALTER TABLE signoz_traces.signoz_spans ON CLUSTER {{.SIGNOZ_CLUSTER}} MODIFY SETTING ttl_only_drop_parts = 1; +ALTER TABLE signoz_traces.durationSort ON CLUSTER {{.SIGNOZ_CLUSTER}} MODIFY SETTING ttl_only_drop_parts = 1; +ALTER TABLE signoz_traces.dependency_graph_minutes ON CLUSTER {{.SIGNOZ_CLUSTER}} MODIFY SETTING ttl_only_drop_parts = 1; +ALTER TABLE signoz_traces.usage_explorer ON CLUSTER {{.SIGNOZ_CLUSTER}} MODIFY SETTING ttl_only_drop_parts = 1; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000020_resource_att_exceptions.down.sql b/migrator/migrators/traces/migrations/000020_resource_att_exceptions.down.sql similarity index 58% rename from exporter/clickhousetracesexporter/migrations/000020_resource_att_exceptions.down.sql rename to migrator/migrators/traces/migrations/000020_resource_att_exceptions.down.sql index 50619846..9a6f6317 100644 --- a/exporter/clickhousetracesexporter/migrations/000020_resource_att_exceptions.down.sql +++ b/migrator/migrators/traces/migrations/000020_resource_att_exceptions.down.sql @@ -1,4 +1,4 @@ -ALTER TABLE signoz_traces.signoz_error_index_v2 ON CLUSTER cluster +ALTER TABLE signoz_traces.signoz_error_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP COLUMN IF EXISTS resourceTagsMap; -ALTER TABLE signoz_traces.distributed_signoz_error_index_v2 ON CLUSTER cluster +ALTER TABLE signoz_traces.distributed_signoz_error_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP COLUMN IF EXISTS resourceTagsMap; \ No newline at end of file diff --git a/exporter/clickhousetracesexporter/migrations/000020_resource_att_exceptions.up.sql b/migrator/migrators/traces/migrations/000020_resource_att_exceptions.up.sql similarity index 82% rename from exporter/clickhousetracesexporter/migrations/000020_resource_att_exceptions.up.sql rename to migrator/migrators/traces/migrations/000020_resource_att_exceptions.up.sql index eb7e55b0..f6d2909f 100644 --- a/exporter/clickhousetracesexporter/migrations/000020_resource_att_exceptions.up.sql +++ b/migrator/migrators/traces/migrations/000020_resource_att_exceptions.up.sql @@ -1,7 +1,7 @@ -ALTER TABLE signoz_traces.signoz_error_index_v2 ON CLUSTER cluster +ALTER TABLE signoz_traces.signoz_error_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD COLUMN IF NOT EXISTS resourceTagsMap Map(LowCardinality(String), String) CODEC(ZSTD(1)), ADD INDEX IF NOT EXISTS idx_resourceTagsMapKeys mapKeys(resourceTagsMap) TYPE bloom_filter(0.01) GRANULARITY 64, ADD INDEX IF NOT EXISTS idx_resourceTagsMapValues mapValues(resourceTagsMap) TYPE bloom_filter(0.01) GRANULARITY 64; -ALTER TABLE signoz_traces.distributed_signoz_error_index_v2 ON CLUSTER cluster +ALTER TABLE signoz_traces.distributed_signoz_error_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD COLUMN IF NOT EXISTS resourceTagsMap Map(LowCardinality(String), String) CODEC(ZSTD(1)); diff --git a/exporter/clickhousetracesexporter/migrations/000021_resource_attributes_signoz_index_v2.down.sql b/migrator/migrators/traces/migrations/000021_resource_attributes_signoz_index_v2.down.sql similarity index 61% rename from exporter/clickhousetracesexporter/migrations/000021_resource_attributes_signoz_index_v2.down.sql rename to migrator/migrators/traces/migrations/000021_resource_attributes_signoz_index_v2.down.sql index f9a758b5..6c4a90db 100644 --- a/exporter/clickhousetracesexporter/migrations/000021_resource_attributes_signoz_index_v2.down.sql +++ b/migrator/migrators/traces/migrations/000021_resource_attributes_signoz_index_v2.down.sql @@ -1,4 +1,4 @@ -ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER cluster +ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP COLUMN IF EXISTS resourceTagsMap; -ALTER TABLE signoz_traces.distributed_signoz_index_v2 ON CLUSTER cluster +ALTER TABLE signoz_traces.distributed_signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP COLUMN IF EXISTS resourceTagsMap; diff --git a/exporter/clickhousetracesexporter/migrations/000021_resource_attributes_signoz_index_v2.up.sql b/migrator/migrators/traces/migrations/000021_resource_attributes_signoz_index_v2.up.sql similarity index 84% rename from exporter/clickhousetracesexporter/migrations/000021_resource_attributes_signoz_index_v2.up.sql rename to migrator/migrators/traces/migrations/000021_resource_attributes_signoz_index_v2.up.sql index ab334a2c..cf63e7b3 100644 --- a/exporter/clickhousetracesexporter/migrations/000021_resource_attributes_signoz_index_v2.up.sql +++ b/migrator/migrators/traces/migrations/000021_resource_attributes_signoz_index_v2.up.sql @@ -1,7 +1,7 @@ -ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER cluster +ALTER TABLE signoz_traces.signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD COLUMN IF NOT EXISTS resourceTagsMap Map(LowCardinality(String), String) CODEC(ZSTD(1)), ADD INDEX IF NOT EXISTS idx_resourceTagsMapKeys mapKeys(resourceTagsMap) TYPE bloom_filter(0.01) GRANULARITY 64, ADD INDEX IF NOT EXISTS idx_resourceTagsMapValues mapValues(resourceTagsMap) TYPE bloom_filter(0.01) GRANULARITY 64; -ALTER TABLE signoz_traces.distributed_signoz_index_v2 ON CLUSTER cluster +ALTER TABLE signoz_traces.distributed_signoz_index_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD COLUMN IF NOT EXISTS resourceTagsMap Map(LowCardinality(String), String) CODEC(ZSTD(1)); diff --git a/exporter/clickhousetracesexporter/migrations/000022_dependency_graph_minutes_v2.down.sql b/migrator/migrators/traces/migrations/000022_dependency_graph_minutes_v2.down.sql similarity index 60% rename from exporter/clickhousetracesexporter/migrations/000022_dependency_graph_minutes_v2.down.sql rename to migrator/migrators/traces/migrations/000022_dependency_graph_minutes_v2.down.sql index beeb0017..8145bcc5 100644 --- a/exporter/clickhousetracesexporter/migrations/000022_dependency_graph_minutes_v2.down.sql +++ b/migrator/migrators/traces/migrations/000022_dependency_graph_minutes_v2.down.sql @@ -1,6 +1,6 @@ -DROP TABLE IF EXISTS signoz_traces.dependency_graph_minutes_v2 ON CLUSTER cluster; -DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_service_calls_mv_v2 ON CLUSTER cluster; -DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_db_calls_mv_v2 ON CLUSTER cluster; -DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_messaging_calls_mv_v2 ON CLUSTER cluster; +DROP TABLE IF EXISTS signoz_traces.dependency_graph_minutes_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_service_calls_mv_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_db_calls_mv_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}}; +DROP VIEW IF EXISTS signoz_traces.dependency_graph_minutes_messaging_calls_mv_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}}; -DROP TABLE IF EXISTS signoz_traces.distributed_dependency_graph_minutes_v2 ON CLUSTER cluster; +DROP TABLE IF EXISTS signoz_traces.distributed_dependency_graph_minutes_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}}; diff --git a/exporter/clickhousetracesexporter/migrations/000022_dependency_graph_minutes_v2.up.sql b/migrator/migrators/traces/migrations/000022_dependency_graph_minutes_v2.up.sql similarity index 91% rename from exporter/clickhousetracesexporter/migrations/000022_dependency_graph_minutes_v2.up.sql rename to migrator/migrators/traces/migrations/000022_dependency_graph_minutes_v2.up.sql index 4e6ff652..c2410357 100644 --- a/exporter/clickhousetracesexporter/migrations/000022_dependency_graph_minutes_v2.up.sql +++ b/migrator/migrators/traces/migrations/000022_dependency_graph_minutes_v2.up.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS signoz_traces.dependency_graph_minutes_v2 ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_traces.dependency_graph_minutes_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} ( src LowCardinality(String) CODEC(ZSTD(1)), dest LowCardinality(String) CODEC(ZSTD(1)), duration_quantiles_state AggregateFunction(quantiles(0.5, 0.75, 0.9, 0.95, 0.99), Float64) CODEC(Default), @@ -14,7 +14,7 @@ ORDER BY (timestamp, src, dest, deployment_environment, k8s_cluster_name, k8s_na TTL toDateTime(timestamp) + INTERVAL 604800 SECOND DELETE; -CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_service_calls_mv_v2 ON CLUSTER cluster +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_service_calls_mv_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_traces.dependency_graph_minutes_v2 AS SELECT A.serviceName as src, @@ -30,7 +30,7 @@ FROM signoz_traces.signoz_index_v2 AS A, signoz_traces.signoz_index_v2 AS B WHERE (A.serviceName != B.serviceName) AND (A.spanID = B.parentSpanID) GROUP BY timestamp, src, dest, deployment_environment, k8s_cluster_name, k8s_namespace_name; -CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_db_calls_mv_v2 ON CLUSTER cluster +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_db_calls_mv_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_traces.dependency_graph_minutes_v2 AS SELECT serviceName as src, @@ -46,7 +46,7 @@ FROM signoz_traces.signoz_index_v2 WHERE dest != '' and kind != 2 GROUP BY timestamp, src, dest, deployment_environment, k8s_cluster_name, k8s_namespace_name; -CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_messaging_calls_mv_v2 ON CLUSTER cluster +CREATE MATERIALIZED VIEW IF NOT EXISTS signoz_traces.dependency_graph_minutes_messaging_calls_mv_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} TO signoz_traces.dependency_graph_minutes_v2 AS SELECT serviceName as src, @@ -62,5 +62,5 @@ FROM signoz_traces.signoz_index_v2 WHERE dest != '' and kind != 2 GROUP BY timestamp, src, dest, deployment_environment, k8s_cluster_name, k8s_namespace_name; -CREATE TABLE IF NOT EXISTS signoz_traces.distributed_dependency_graph_minutes_v2 ON CLUSTER cluster AS signoz_traces.dependency_graph_minutes_v2 +CREATE TABLE IF NOT EXISTS signoz_traces.distributed_dependency_graph_minutes_v2 ON CLUSTER {{.SIGNOZ_CLUSTER}} AS signoz_traces.dependency_graph_minutes_v2 ENGINE = Distributed("cluster", "signoz_traces", dependency_graph_minutes_v2, cityHash64(rand())); diff --git a/migrator/migrators/traces/migrations/000023_span_attributes_mv.down.sql b/migrator/migrators/traces/migrations/000023_span_attributes_mv.down.sql new file mode 100644 index 00000000..ed7f1017 --- /dev/null +++ b/migrator/migrators/traces/migrations/000023_span_attributes_mv.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS signoz_traces.distributed_span_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} +DROP TABLE IF EXISTS signoz_traces.span_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}}; diff --git a/exporter/clickhousetracesexporter/migrations/000023_span_attributes_mv.up.sql b/migrator/migrators/traces/migrations/000023_span_attributes_mv.up.sql similarity index 89% rename from exporter/clickhousetracesexporter/migrations/000023_span_attributes_mv.up.sql rename to migrator/migrators/traces/migrations/000023_span_attributes_mv.up.sql index fc942c43..ac3a6e49 100644 --- a/exporter/clickhousetracesexporter/migrations/000023_span_attributes_mv.up.sql +++ b/migrator/migrators/traces/migrations/000023_span_attributes_mv.up.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS signoz_traces.span_attributes ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_traces.span_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} ( timestamp DateTime CODEC(DoubleDelta, ZSTD(1)), tagKey LowCardinality(String) CODEC(ZSTD(1)), tagType Enum('tag', 'resource') CODEC(ZSTD(1)), @@ -11,5 +11,5 @@ ORDER BY (tagKey, tagType, dataType, stringTagValue, float64TagValue, isColumn) TTL toDateTime(timestamp) + INTERVAL 172800 SECOND DELETE SETTINGS ttl_only_drop_parts = 1, allow_nullable_key = 1; -CREATE TABLE IF NOT EXISTS signoz_traces.distributed_span_attributes ON CLUSTER cluster AS signoz_traces.span_attributes +CREATE TABLE IF NOT EXISTS signoz_traces.distributed_span_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} AS signoz_traces.span_attributes ENGINE = Distributed("cluster", "signoz_traces", span_attributes, cityHash64(rand())); diff --git a/exporter/clickhousetracesexporter/migrations/000024_span_attributes_keys.down.sql b/migrator/migrators/traces/migrations/000024_span_attributes_keys.down.sql similarity index 68% rename from exporter/clickhousetracesexporter/migrations/000024_span_attributes_keys.down.sql rename to migrator/migrators/traces/migrations/000024_span_attributes_keys.down.sql index bb1600cd..8ce0233c 100644 --- a/exporter/clickhousetracesexporter/migrations/000024_span_attributes_keys.down.sql +++ b/migrator/migrators/traces/migrations/000024_span_attributes_keys.down.sql @@ -1,2 +1,2 @@ -DROP TABLE IF EXISTS signoz_traces.distributed_span_attributes_keys ON CLUSTER cluster -DROP TABLE IF EXISTS signoz_traces.span_attributes_keys ON CLUSTER cluster; +DROP TABLE IF EXISTS signoz_traces.distributed_span_attributes_keys ON CLUSTER {{.SIGNOZ_CLUSTER}} +DROP TABLE IF EXISTS signoz_traces.span_attributes_keys ON CLUSTER {{.SIGNOZ_CLUSTER}}; diff --git a/exporter/clickhousetracesexporter/migrations/000024_span_attributes_keys.up.sql b/migrator/migrators/traces/migrations/000024_span_attributes_keys.up.sql similarity index 81% rename from exporter/clickhousetracesexporter/migrations/000024_span_attributes_keys.up.sql rename to migrator/migrators/traces/migrations/000024_span_attributes_keys.up.sql index 4dd8d505..f2565df3 100644 --- a/exporter/clickhousetracesexporter/migrations/000024_span_attributes_keys.up.sql +++ b/migrator/migrators/traces/migrations/000024_span_attributes_keys.up.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS signoz_traces.span_attributes_keys ON CLUSTER cluster ( +CREATE TABLE IF NOT EXISTS signoz_traces.span_attributes_keys ON CLUSTER {{.SIGNOZ_CLUSTER}} ( tagKey LowCardinality(String) CODEC(ZSTD(1)), tagType Enum('tag', 'resource') CODEC(ZSTD(1)), dataType Enum('string', 'bool', 'float64') CODEC(ZSTD(1)), @@ -6,5 +6,5 @@ CREATE TABLE IF NOT EXISTS signoz_traces.span_attributes_keys ON CLUSTER cluster ) ENGINE ReplacingMergeTree ORDER BY (tagKey, tagType, dataType, isColumn); -CREATE TABLE IF NOT EXISTS signoz_traces.distributed_span_attributes_keys ON CLUSTER cluster AS signoz_traces.span_attributes_keys +CREATE TABLE IF NOT EXISTS signoz_traces.distributed_span_attributes_keys ON CLUSTER {{.SIGNOZ_CLUSTER}} AS signoz_traces.span_attributes_keys ENGINE = Distributed("cluster", "signoz_traces", span_attributes_keys, cityHash64(rand())); diff --git a/migrator/migrators/traces/migrator.go b/migrator/migrators/traces/migrator.go new file mode 100644 index 00000000..6153ac40 --- /dev/null +++ b/migrator/migrators/traces/migrator.go @@ -0,0 +1,25 @@ +package traces + +import ( + "context" + + basemigrator "github.com/SigNoz/signoz-otel-collector/migrator/migrators/baseMigrator" +) + +const ( + name = "traces" + database = "signoz_traces" + migrationFolder = "migrator/migrators/traces/migrations" +) + +type TracesMigrator struct { + *basemigrator.BaseMigrator +} + +func (m *TracesMigrator) Migrate(ctx context.Context) error { + return m.BaseMigrator.Migrate(ctx, database, migrationFolder) +} + +func (m *TracesMigrator) Name() string { + return name +} \ No newline at end of file