Skip to content

Commit

Permalink
fix: update logs migrations to remove issues (#346)
Browse files Browse the repository at this point in the history
* fix: update logs migrations to remove issues

* fix: add old migration data
  • Loading branch information
nityanandagohain committed Jul 29, 2024
1 parent c6f81e7 commit d3ed24f
Show file tree
Hide file tree
Showing 9 changed files with 80 additions and 61 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ CREATE TABLE IF NOT EXISTS signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} (
attributes_int64_value Array(Int64) CODEC(ZSTD(1)),
attributes_float64_key Array(String) CODEC(ZSTD(1)),
attributes_float64_value Array(Float64) CODEC(ZSTD(1)),
INDEX body_idx body TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4
INDEX body_idx body TYPE ngrambf_v1(4, 60000, 5, 0) GRANULARITY 1
) ENGINE {{.SIGNOZ_REPLICATED}}MergeTree
PARTITION BY toDate(timestamp / 1000000000)
ORDER BY (timestamp, id)
Expand Down
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
ALTER TABLE signoz_logs.logs ON CLUSTER {{ .SIGNOZ_CLUSTER }} DROP INDEX IF EXISTS body_idx;
ALTER TABLE signoz_logs.logs ON CLUSTER {{ .SIGNOZ_CLUSTER }} ADD INDEX IF NOT EXISTS body_idx body TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4;
-- ALTER TABLE signoz_logs.logs ON CLUSTER {{ .SIGNOZ_CLUSTER }} DROP INDEX IF EXISTS body_idx
-- ALTER TABLE signoz_logs.logs ON CLUSTER {{ .SIGNOZ_CLUSTER }} ADD INDEX IF NOT EXISTS body_idx body TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4

SELECT 1
Original file line number Diff line number Diff line change
@@ -1,2 +1,5 @@
ALTER TABLE signoz_logs.logs ON CLUSTER {{ .SIGNOZ_CLUSTER }} DROP INDEX IF EXISTS body_idx ;
ALTER TABLE signoz_logs.logs ON CLUSTER {{ .SIGNOZ_CLUSTER }} ADD INDEX IF NOT EXISTS body_idx lower(body) TYPE ngrambf_v1(4, 60000, 5, 0) GRANULARITY 1;
-- Please run the below commands if you are trying to fix schema migration issue https://signoz.io/docs/userguide/logs_troubleshooting/#schema-migrator-dirty-database-version
-- ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP INDEX IF EXISTS body_idx
-- ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD INDEX IF NOT EXISTS body_idx lower(body) TYPE ngrambf_v1(4, 60000, 5, 0) GRANULARITY 1SELECT 1

SELECT 1
Original file line number Diff line number Diff line change
@@ -1,18 +1,19 @@
-- Please run the below commands if you are trying to fix schema migration issue https://signoz.io/docs/userguide/logs_troubleshooting/#schema-migrator-dirty-database-version
ALTER TABLE signoz_logs.tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} modify column tagType Enum8('tag', 'resource') CODEC(ZSTD(1));
ALTER TABLE signoz_logs.distributed_tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} modify column tagType Enum8('tag', 'resource') CODEC(ZSTD(1));


ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP INDEX IF EXISTS instrumentation_scope_idx;
ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP INDEX IF EXISTS scope_name_idx;

ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP column IF EXISTS instrumentation_scope;
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP column IF EXISTS instrumentation_scope;
ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP column IF EXISTS scope_name;
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP column IF EXISTS scope_name;

ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP column IF EXISTS instrumentation_scope_version;
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP column IF EXISTS instrumentation_scope_version;
ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP column IF EXISTS scope_version;
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP column IF EXISTS scope_version;


ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP column IF EXISTS instrumentation_scope_attributes_string_key;
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP column IF EXISTS instrumentation_scope_attributes_string_key;
ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP column IF EXISTS scope_string_key;
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP column IF EXISTS scope_string_key;

ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP column IF EXISTS instrumentation_scope_attributes_string_value;
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP column IF EXISTS instrumentation_scope_attributes_string_value;
ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP column IF EXISTS scope_string_value;
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} DROP column IF EXISTS scope_string_value;
Original file line number Diff line number Diff line change
@@ -1,18 +1,33 @@
ALTER TABLE signoz_logs.tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} modify column tagType Enum8('tag', 'resource', 'instrumentation_scope') CODEC(ZSTD(1));
ALTER TABLE signoz_logs.distributed_tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} modify column tagType Enum8('tag', 'resource', 'instrumentation_scope') CODEC(ZSTD(1));
-- Please run the below commands if you are trying to fix schema migration issue https://signoz.io/docs/userguide/logs_troubleshooting/#schema-migrator-dirty-database-version
-- ALTER TABLE signoz_logs.tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} modify column tagType Enum8('tag', 'resource', 'instrumentation_scope') CODEC(ZSTD(1))
-- ALTER TABLE signoz_logs.distributed_tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} modify column tagType Enum8('tag', 'resource', 'instrumentation_scope') CODEC(ZSTD(1))
-- ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS instrumentation_scope String CODEC(ZSTD(1))
-- ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS instrumentation_scope String CODEC(ZSTD(1))
-- ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS instrumentation_scope_version String CODEC(ZSTD(1))
-- ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS instrumentation_scope_version String CODEC(ZSTD(1))
-- ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS instrumentation_scope_attributes_string_key Array(String) CODEC(ZSTD(1))
-- ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS instrumentation_scope_attributes_string_key Array(String) CODEC(ZSTD(1))
-- ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS instrumentation_scope_attributes_string_value Array(String) CODEC(ZSTD(1))
-- ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS instrumentation_scope_attributes_string_value Array(String) CODEC(ZSTD(1))
-- ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD INDEX IF NOT EXISTS instrumentation_scope_idx (instrumentation_scope) TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4


ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS instrumentation_scope String CODEC(ZSTD(1));
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS instrumentation_scope String CODEC(ZSTD(1));
-- Don't run the commands below
ALTER TABLE signoz_logs.tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} modify column tagType Enum8('tag', 'resource', 'scope') CODEC(ZSTD(1));
ALTER TABLE signoz_logs.distributed_tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} modify column tagType Enum8('tag', 'resource', 'scope') CODEC(ZSTD(1));

ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS instrumentation_scope_version String CODEC(ZSTD(1));
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS instrumentation_scope_version String CODEC(ZSTD(1));

ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS instrumentation_scope_attributes_string_key Array(String) CODEC(ZSTD(1));
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS instrumentation_scope_attributes_string_key Array(String) CODEC(ZSTD(1));
ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS scope_name String CODEC(ZSTD(1));
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS scope_name String CODEC(ZSTD(1));

ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS instrumentation_scope_attributes_string_value Array(String) CODEC(ZSTD(1));
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS instrumentation_scope_attributes_string_value Array(String) CODEC(ZSTD(1));
ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS scope_version String CODEC(ZSTD(1));
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS scope_version String CODEC(ZSTD(1));

ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS scope_string_key Array(String) CODEC(ZSTD(1));
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS scope_string_key Array(String) CODEC(ZSTD(1));

ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD INDEX IF NOT EXISTS instrumentation_scope_idx (instrumentation_scope) TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4;
ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS scope_string_value Array(String) CODEC(ZSTD(1));
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD column IF NOT EXISTS scope_string_value Array(String) CODEC(ZSTD(1));


ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD INDEX IF NOT EXISTS scope_name_idx (scope_name) TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4;
Original file line number Diff line number Diff line change
@@ -1,16 +1,12 @@
ALTER TABLE signoz_logs.logs DROP INDEX IF EXISTS scope_name_idx;

ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS scope_name to instrumentation_scope;
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS scope_name to instrumentation_scope;

ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS scope_version to instrumentation_scope_version;
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS scope_version to instrumentation_scope_version;

ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS scope_string_key to instrumentation_scope_attributes_string_key;
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS scope_string_key to instrumentation_scope_attributes_string_key;

ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS scope_string_value to instrumentation_scope_attributes_string_value;
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS scope_string_value to instrumentation_scope_attributes_string_value;


ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD INDEX IF NOT EXISTS instrumentation_scope_idx (instrumentation_scope) TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4;
--ALTER TABLE signoz_logs.logs DROP INDEX IF EXISTS scope_name_idx
--ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS scope_name to instrumentation_scope
--ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS scope_name to instrumentation_scope
--ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS scope_version to instrumentation_scope_version
--ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS scope_version to instrumentation_scope_version
--ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS scope_string_key to instrumentation_scope_attributes_string_key
--ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS scope_string_key to instrumentation_scope_attributes_string_key
--ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS scope_string_value to instrumentation_scope_attributes_string_value
--ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS scope_string_value to instrumentation_scope_attributes_string_value
--ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD INDEX IF NOT EXISTS instrumentation_scope_idx (instrumentation_scope) TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4

SELECT 1
Original file line number Diff line number Diff line change
@@ -1,16 +1,13 @@
ALTER TABLE signoz_logs.logs DROP INDEX IF EXISTS instrumentation_scope_idx;

ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS instrumentation_scope to scope_name;
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS instrumentation_scope to scope_name;

ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS instrumentation_scope_version to scope_version;
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS instrumentation_scope_version to scope_version;

ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS instrumentation_scope_attributes_string_key to scope_string_key;
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS instrumentation_scope_attributes_string_key to scope_string_key;

ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS instrumentation_scope_attributes_string_value to scope_string_value;
ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS instrumentation_scope_attributes_string_value to scope_string_value;


ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD INDEX IF NOT EXISTS scope_name_idx (scope_name) TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4;
-- Please run the below commands if you are trying to fix schema migration issue https://signoz.io/docs/userguide/logs_troubleshooting/#schema-migrator-dirty-database-version
--ALTER TABLE signoz_logs.logs DROP INDEX IF EXISTS instrumentation_scope_idx
--ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS instrumentation_scope to scope_name
--ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS instrumentation_scope to scope_name
--ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS instrumentation_scope_version to scope_version
--ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS instrumentation_scope_version to scope_version
--ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS instrumentation_scope_attributes_string_key to scope_string_key
--ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS instrumentation_scope_attributes_string_key to scope_string_key
--ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS instrumentation_scope_attributes_string_value to scope_string_value
--ALTER TABLE signoz_logs.distributed_logs ON CLUSTER {{.SIGNOZ_CLUSTER}} RENAME column IF EXISTS instrumentation_scope_attributes_string_value to scope_string_value
--ALTER TABLE signoz_logs.logs ON CLUSTER {{.SIGNOZ_CLUSTER}} ADD INDEX IF NOT EXISTS scope_name_idx (scope_name) TYPE tokenbf_v1(10240, 3, 0) GRANULARITY 4

SELECT 1
Original file line number Diff line number Diff line change
@@ -1,2 +1,4 @@
ALTER TABLE signoz_logs.tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} modify column tagType Enum8('tag', 'resource', 'instrumentation_scope') CODEC(ZSTD(1));
ALTER TABLE signoz_logs.distributed_tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} modify column tagType Enum8('tag', 'resource', 'instrumentation_scope') CODEC(ZSTD(1));
--ALTER TABLE signoz_logs.tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} modify column tagType Enum8('tag', 'resource', 'instrumentation_scope') CODEC(ZSTD(1))
--ALTER TABLE signoz_logs.distributed_tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} modify column tagType Enum8('tag', 'resource', 'instrumentation_scope') CODEC(ZSTD(1))

SELECT 1
Original file line number Diff line number Diff line change
@@ -1,2 +1,5 @@
ALTER TABLE signoz_logs.tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} modify column tagType Enum8('tag', 'resource', 'scope') CODEC(ZSTD(1));
ALTER TABLE signoz_logs.distributed_tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} modify column tagType Enum8('tag', 'resource', 'scope') CODEC(ZSTD(1));
-- Please run the below commands if you are trying to fix schema migration issue https://signoz.io/docs/userguide/logs_troubleshooting/#schema-migrator-dirty-database-version
--ALTER TABLE signoz_logs.tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} modify column tagType Enum8('tag', 'resource', 'scope') CODEC(ZSTD(1))
--ALTER TABLE signoz_logs.distributed_tag_attributes ON CLUSTER {{.SIGNOZ_CLUSTER}} modify column tagType Enum8('tag', 'resource', 'scope') CODEC(ZSTD(1))

SELECT 1

0 comments on commit d3ed24f

Please sign in to comment.