From 67d484b30d906dcc4562fb8512eb98387a966018 Mon Sep 17 00:00:00 2001 From: Tillman Elser Date: Tue, 16 Jul 2024 20:18:43 -0700 Subject: [PATCH 1/7] drop old grouping_records table --- ...b647e9a_drop_old_grouping_records_table.py | 38 +++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py diff --git a/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py b/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py new file mode 100644 index 000000000..ee624e33a --- /dev/null +++ b/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py @@ -0,0 +1,38 @@ +"""drop old grouping_records table + +Revision ID: 2597db647e9a +Revises: a0d00121d118 +Create Date: 2024-07-17 03:16:52.924194 + +""" + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "2597db647e9a" +down_revision = "a0d00121d118" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_table("grouping_records_old") + + +def downgrade(): + op.create_table( + "grouping_records_old", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("group_id", sa.Integer(), nullable=False), + sa.Column("record_id", sa.Integer(), nullable=False), + sa.PrimaryKeyConstraint("id"), + sa.ForeignKeyConstraint( + ["group_id"], + ["groups.id"], + ), + sa.ForeignKeyConstraint( + ["record_id"], + ["records.id"], + ), + ) From 625f7f7b1d9aac7726a20c0d9f0df3cc70cd8acc Mon Sep 17 00:00:00 2001 From: Tillman Elser Date: Tue, 16 Jul 2024 20:20:25 -0700 Subject: [PATCH 2/7] update upgrade/downgrade logic --- ...b647e9a_drop_old_grouping_records_table.py | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py b/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py index ee624e33a..bc3bea70c 100644 --- a/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py +++ b/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py @@ -6,7 +6,6 @@ """ -import sqlalchemy as sa from alembic import op # revision identifiers, used by Alembic. @@ -17,22 +16,8 @@ def upgrade(): - op.drop_table("grouping_records_old") + op.execute("DROP TABLE IF EXISTS grouping_records_old") def downgrade(): - op.create_table( - "grouping_records_old", - sa.Column("id", sa.Integer(), nullable=False), - sa.Column("group_id", sa.Integer(), nullable=False), - sa.Column("record_id", sa.Integer(), nullable=False), - sa.PrimaryKeyConstraint("id"), - sa.ForeignKeyConstraint( - ["group_id"], - ["groups.id"], - ), - sa.ForeignKeyConstraint( - ["record_id"], - ["records.id"], - ), - ) + pass From 0b94109acbb8e27a3043187cd27360ef9e711d18 Mon Sep 17 00:00:00 2001 From: Tillman Elser Date: Tue, 16 Jul 2024 20:31:23 -0700 Subject: [PATCH 3/7] add renames to second migration --- ...597db647e9a_drop_old_grouping_records_table.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py b/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py index bc3bea70c..7c3c4f530 100644 --- a/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py +++ b/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py @@ -16,8 +16,21 @@ def upgrade(): + op.execute("ALTER TABLE IF EXISTS grouping_records RENAME to grouping_records_old;") + + op.execute("ALTER TABLE grouping_records_new RENAME TO grouping_records;") + for i in range(100): + op.execute(f"ALTER TABLE grouping_records_new_p{i} RENAME TO grouping_records_p{i};") + op.execute("DROP TABLE IF EXISTS grouping_records_old") def downgrade(): - pass + op.execute("ALTER TABLE IF EXISTS grouping_records RENAME TO grouping_records_new;") + + for i in range(100): + op.execute(f"ALTER TABLE grouping_records_p{i} RENAME TO grouping_records_new_p{i};") + + op.execute("ALTER TABLE grouping_records_old RENAME TO grouping_records;") + + op.execute("DROP TABLE IF EXISTS grouping_records_new") From 1a34dba0c6efc6e406c0218a8fa63fa76bd0ba1e Mon Sep 17 00:00:00 2001 From: Tillman Elser Date: Tue, 16 Jul 2024 20:35:01 -0700 Subject: [PATCH 4/7] merge other migration branch --- ...b647e9a_drop_old_grouping_records_table.py | 2 +- .../versions/d87a6410efe4_migration.py | 68 +++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 src/migrations/versions/d87a6410efe4_migration.py diff --git a/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py b/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py index 7c3c4f530..cb827dfc9 100644 --- a/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py +++ b/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py @@ -10,7 +10,7 @@ # revision identifiers, used by Alembic. revision = "2597db647e9a" -down_revision = "a0d00121d118" +down_revision = "d87a6410efe4" branch_labels = None depends_on = None diff --git a/src/migrations/versions/d87a6410efe4_migration.py b/src/migrations/versions/d87a6410efe4_migration.py new file mode 100644 index 000000000..d42e1786e --- /dev/null +++ b/src/migrations/versions/d87a6410efe4_migration.py @@ -0,0 +1,68 @@ +"""Update HNSW parameters for grouping_records + +Revision ID: d87a6410efe4 +Revises: a0d00121d118 +Create Date: 2024-07-09 22:28:26.035785 + +""" + +from alembic import op + +# revision identifiers, used by Alembic. +revision = "d87a6410efe4" +down_revision = "a0d00121d118" +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute("DROP TABLE IF EXISTS grouping_records_new CASCADE;") + + op.execute( + """ + CREATE TABLE grouping_records_new ( + id INTEGER NOT NULL, + project_id BIGINT NOT NULL, + hash VARCHAR(32) NOT NULL, + message VARCHAR NOT NULL, + error_type VARCHAR, + stacktrace_embedding VECTOR(768) NOT NULL, + PRIMARY KEY (id, project_id) + ) PARTITION BY HASH (project_id); + """ + ) + + for i in range(100): + op.execute( + f""" + CREATE TABLE grouping_records_new_p{i} PARTITION OF grouping_records_new + FOR VALUES WITH (MODULUS 100, REMAINDER {i}); + """ + ) + + op.execute( + """ + INSERT INTO grouping_records_new (id, project_id, message, error_type, stacktrace_embedding, hash) + SELECT id, project_id, message, error_type, stacktrace_embedding, hash + FROM grouping_records; + """ + ) + + op.execute( + """ + CREATE INDEX IF NOT EXISTS ix_grouping_records_new_stacktrace_embedding_hnsw + ON grouping_records_new USING hnsw (stacktrace_embedding vector_cosine_ops) + WITH (m = 16, ef_construction = 200); + """ + ) + + op.execute( + "CREATE INDEX IF NOT EXISTS ix_grouping_records_project_id ON grouping_records_new (project_id);" + ) + + with op.batch_alter_table("grouping_records_new", schema=None) as batch_op: + batch_op.create_unique_constraint("u_project_id_hash_composite", ["project_id", "hash"]) + + +def downgrade(): + op.execute("DROP TABLE IF EXISTS grouping_records_new CASCADE;") From 9a2a8a78b9ae3258df7ca3580b67198e0855ef93 Mon Sep 17 00:00:00 2001 From: Tillman Elser Date: Tue, 16 Jul 2024 20:36:17 -0700 Subject: [PATCH 5/7] update comment --- .../versions/2597db647e9a_drop_old_grouping_records_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py b/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py index cb827dfc9..dc91824f5 100644 --- a/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py +++ b/src/migrations/versions/2597db647e9a_drop_old_grouping_records_table.py @@ -1,7 +1,7 @@ """drop old grouping_records table Revision ID: 2597db647e9a -Revises: a0d00121d118 +Revises: d87a6410efe4 Create Date: 2024-07-17 03:16:52.924194 """ From 867c77ad764b6d3782e7758a8e10b26f0bead69d Mon Sep 17 00:00:00 2001 From: Tillman Elser Date: Wed, 17 Jul 2024 11:15:21 -0700 Subject: [PATCH 6/7] make migration fail on index creation failure --- src/migrations/versions/d87a6410efe4_migration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/migrations/versions/d87a6410efe4_migration.py b/src/migrations/versions/d87a6410efe4_migration.py index d42e1786e..12cc8d1fa 100644 --- a/src/migrations/versions/d87a6410efe4_migration.py +++ b/src/migrations/versions/d87a6410efe4_migration.py @@ -50,14 +50,14 @@ def upgrade(): op.execute( """ - CREATE INDEX IF NOT EXISTS ix_grouping_records_new_stacktrace_embedding_hnsw + CREATE INDEX ix_grouping_records_new_stacktrace_embedding_hnsw ON grouping_records_new USING hnsw (stacktrace_embedding vector_cosine_ops) WITH (m = 16, ef_construction = 200); """ ) op.execute( - "CREATE INDEX IF NOT EXISTS ix_grouping_records_project_id ON grouping_records_new (project_id);" + "CREATE INDEX ix_grouping_records_new_project_id ON grouping_records_new (project_id);" ) with op.batch_alter_table("grouping_records_new", schema=None) as batch_op: From e2b67bc64633d6d5f115301219a274b45f26682c Mon Sep 17 00:00:00 2001 From: Tillman Elser Date: Wed, 17 Jul 2024 14:07:29 -0700 Subject: [PATCH 7/7] int -> bigint --- src/migrations/versions/d87a6410efe4_migration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/migrations/versions/d87a6410efe4_migration.py b/src/migrations/versions/d87a6410efe4_migration.py index 12cc8d1fa..25f970148 100644 --- a/src/migrations/versions/d87a6410efe4_migration.py +++ b/src/migrations/versions/d87a6410efe4_migration.py @@ -21,7 +21,7 @@ def upgrade(): op.execute( """ CREATE TABLE grouping_records_new ( - id INTEGER NOT NULL, + id BIGINT NOT NULL, project_id BIGINT NOT NULL, hash VARCHAR(32) NOT NULL, message VARCHAR NOT NULL,