From ce4bd13119b24d32e2358a291f2a46f31f799efe Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Tue, 17 Sep 2024 20:21:08 -0400 Subject: [PATCH] fix the test? --- .../entities/spans_num_attrs.yaml | 4 +- .../entities/spans_str_attrs.yaml | 1 - .../storages/spans_num_attrs.yaml | 8 +-- .../storages/spans_str_attrs.yaml | 5 +- .../0014_span_attribute_table_smaller.py | 70 ++++++++++--------- snuba/web/rpc/trace_item_attribute_values.py | 1 + 6 files changed, 47 insertions(+), 42 deletions(-) diff --git a/snuba/datasets/configuration/events_analytics_platform/entities/spans_num_attrs.yaml b/snuba/datasets/configuration/events_analytics_platform/entities/spans_num_attrs.yaml index d6cf66aaea..623011e089 100644 --- a/snuba/datasets/configuration/events_analytics_platform/entities/spans_num_attrs.yaml +++ b/snuba/datasets/configuration/events_analytics_platform/entities/spans_num_attrs.yaml @@ -6,9 +6,9 @@ schema: [ { name: organization_id, type: UInt, args: { size: 64 } }, { name: project_id, type: UInt, args: { size: 64 } }, - { name: trace_id, type: UUID }, { name: attr_key, type: String }, - { name: attr_value, type: Float, args: { size: 64 } }, + { name: attr_min_value, type: Float, args: { size: 64 } }, + { name: attr_max_value, type: Float, args: { size: 64 } }, { name: timestamp, type: DateTime }, { name: retention_days, type: UInt, args: { size: 16 } }, { diff --git a/snuba/datasets/configuration/events_analytics_platform/entities/spans_str_attrs.yaml b/snuba/datasets/configuration/events_analytics_platform/entities/spans_str_attrs.yaml index ba84e03c48..4de564a375 100644 --- a/snuba/datasets/configuration/events_analytics_platform/entities/spans_str_attrs.yaml +++ b/snuba/datasets/configuration/events_analytics_platform/entities/spans_str_attrs.yaml @@ -6,7 +6,6 @@ schema: [ { name: organization_id, type: UInt, args: { size: 64 } }, { name: project_id, type: UInt, args: { size: 64 } }, - { name: trace_id, type: UUID }, { name: attr_key, type: String }, { name: attr_value, type: String }, { name: timestamp, type: DateTime }, diff --git a/snuba/datasets/configuration/events_analytics_platform/storages/spans_num_attrs.yaml b/snuba/datasets/configuration/events_analytics_platform/storages/spans_num_attrs.yaml index 127e86bb63..580755e442 100644 --- a/snuba/datasets/configuration/events_analytics_platform/storages/spans_num_attrs.yaml +++ b/snuba/datasets/configuration/events_analytics_platform/storages/spans_num_attrs.yaml @@ -13,9 +13,9 @@ schema: [ { name: organization_id, type: UInt, args: { size: 64 } }, { name: project_id, type: UInt, args: { size: 64 } }, - { name: trace_id, type: UUID }, { name: attr_key, type: String }, - { name: attr_value, type: Float, args: { size: 64 } }, + { name: attr_min_value, type: Float, args: { size: 64 } }, + { name: attr_max_value, type: Float, args: { size: 64 } }, { name: timestamp, type: DateTime }, { name: retention_days, type: UInt, args: { size: 16 } }, { @@ -28,8 +28,8 @@ schema: }, }, ] - local_table_name: spans_num_attrs_local - dist_table_name: spans_num_attrs_dist + local_table_name: spans_num_attrs_2_local + dist_table_name: spans_num_attrs_2_dist allocation_policies: - name: ConcurrentRateLimitAllocationPolicy args: diff --git a/snuba/datasets/configuration/events_analytics_platform/storages/spans_str_attrs.yaml b/snuba/datasets/configuration/events_analytics_platform/storages/spans_str_attrs.yaml index 63028acb8c..048a8b6782 100644 --- a/snuba/datasets/configuration/events_analytics_platform/storages/spans_str_attrs.yaml +++ b/snuba/datasets/configuration/events_analytics_platform/storages/spans_str_attrs.yaml @@ -13,7 +13,6 @@ schema: [ { name: organization_id, type: UInt, args: { size: 64 } }, { name: project_id, type: UInt, args: { size: 64 } }, - { name: trace_id, type: UUID }, { name: attr_key, type: String }, { name: attr_value, type: String }, { name: timestamp, type: DateTime }, @@ -28,8 +27,8 @@ schema: }, } ] - local_table_name: spans_str_attrs_local - dist_table_name: spans_str_attrs_dist + local_table_name: spans_str_attrs_2_local + dist_table_name: spans_str_attrs_2_dist allocation_policies: - name: ConcurrentRateLimitAllocationPolicy args: diff --git a/snuba/snuba_migrations/events_analytics_platform/0014_span_attribute_table_smaller.py b/snuba/snuba_migrations/events_analytics_platform/0014_span_attribute_table_smaller.py index a5734b17e3..5fcf99659f 100644 --- a/snuba/snuba_migrations/events_analytics_platform/0014_span_attribute_table_smaller.py +++ b/snuba/snuba_migrations/events_analytics_platform/0014_span_attribute_table_smaller.py @@ -27,9 +27,12 @@ class Migration(migration.ClickhouseNodeMigration): storage_set_key = StorageSetKey.EVENTS_ANALYTICS_PLATFORM granularity = "8192" - str_mv = "spans_str_attrs_mv" - str_local_table = "spans_str_attrs_2_local" - str_dist_table = "spans_str_attrs_2_dist" + old_str_mv = "spans_str_attrs_mv" + old_str_local_table = "spans_str_attrs_local" + old_str_dist_table = "spans_str_attrs_dist" + new_str_mv = "spans_str_attrs_2_mv" + new_str_local_table = "spans_str_attrs_2_local" + new_str_dist_table = "spans_str_attrs_2_dist" str_columns: Sequence[Column[Modifiers]] = [ Column("organization_id", UInt(64)), Column("project_id", UInt(64)), @@ -40,9 +43,12 @@ class Migration(migration.ClickhouseNodeMigration): Column("count", SimpleAggregateFunction("sum", [UInt(64)])), ] - num_mv = "spans_num_attrs_mv" - num_local_table = "spans_num_attrs_2_local" - num_dist_table = "spans_num_attrs_2_dist" + old_num_mv = "spans_num_attrs_mv" + old_num_local_table = "spans_num_attrs_local" + old_num_dist_table = "spans_num_attrs_dist" + new_num_mv = "spans_num_attrs_2_mv" + new_num_local_table = "spans_num_attrs_2_local" + new_num_dist_table = "spans_num_attrs_2_dist" num_columns: Sequence[Column[Modifiers]] = [ Column("organization_id", UInt(64)), Column("project_id", UInt(64)), @@ -56,47 +62,47 @@ class Migration(migration.ClickhouseNodeMigration): def forwards_ops(self) -> Sequence[SqlOperation]: return [ - # first, drop the MVs + # first, drop the old MVs operations.DropTable( storage_set=self.storage_set_key, - table_name=self.str_mv, + table_name=self.old_str_mv, target=OperationTarget.LOCAL, ), operations.DropTable( storage_set=self.storage_set_key, - table_name=self.num_mv, + table_name=self.old_num_mv, target=OperationTarget.LOCAL, ), - # next, drop the dist tables + # next, drop the old dist tables operations.DropTable( storage_set=self.storage_set_key, - table_name=self.str_dist_table, + table_name=self.old_str_dist_table, target=OperationTarget.DISTRIBUTED, ), operations.DropTable( storage_set=self.storage_set_key, - table_name=self.num_dist_table, + table_name=self.old_num_dist_table, target=OperationTarget.DISTRIBUTED, ), - # next, drop the local tables + # next, drop the old local tables operations.DropTable( storage_set=self.storage_set_key, - table_name=self.str_local_table, + table_name=self.old_str_local_table, target=OperationTarget.LOCAL, ), operations.DropTable( storage_set=self.storage_set_key, - table_name=self.num_local_table, + table_name=self.old_num_local_table, target=OperationTarget.LOCAL, ), # now, create new versions of the tables & MVs operations.CreateTable( storage_set=self.storage_set_key, - table_name=self.str_local_table, + table_name=self.new_str_local_table, engine=table_engines.AggregatingMergeTree( storage_set=self.storage_set_key, primary_key="(organization_id, attr_key)", - order_by="(organization_id, attr_key, attr_value, timestamp, project_id, retention_days)", + order_by="(organization_id, attr_key, attr_value, project_id, timestamp, retention_days)", partition_by="toMonday(timestamp)", settings={ "index_granularity": self.granularity, @@ -108,18 +114,18 @@ def forwards_ops(self) -> Sequence[SqlOperation]: ), operations.CreateTable( storage_set=self.storage_set_key, - table_name=self.str_dist_table, + table_name=self.new_str_dist_table, engine=table_engines.Distributed( - local_table_name=self.str_local_table, sharding_key=None + local_table_name=self.new_str_local_table, sharding_key=None ), columns=self.str_columns, target=OperationTarget.DISTRIBUTED, ), operations.CreateMaterializedView( storage_set=self.storage_set_key, - view_name=self.str_mv, + view_name=self.new_str_mv, columns=self.str_columns, - destination_table_name=self.str_local_table, + destination_table_name=self.new_str_local_table, target=OperationTarget.LOCAL, query=f""" SELECT @@ -151,7 +157,7 @@ def forwards_ops(self) -> Sequence[SqlOperation]: ), operations.CreateTable( storage_set=self.storage_set_key, - table_name=self.num_local_table, + table_name=self.new_num_local_table, engine=table_engines.AggregatingMergeTree( storage_set=self.storage_set_key, primary_key="(organization_id, attr_key)", @@ -167,18 +173,18 @@ def forwards_ops(self) -> Sequence[SqlOperation]: ), operations.CreateTable( storage_set=self.storage_set_key, - table_name=self.num_dist_table, + table_name=self.new_num_dist_table, engine=table_engines.Distributed( - local_table_name=self.num_local_table, sharding_key=None + local_table_name=self.new_num_local_table, sharding_key=None ), columns=self.num_columns, target=OperationTarget.DISTRIBUTED, ), operations.CreateMaterializedView( storage_set=self.storage_set_key, - view_name=self.num_mv, + view_name=self.new_num_mv, columns=self.num_columns, - destination_table_name=self.num_local_table, + destination_table_name=self.new_num_local_table, target=OperationTarget.LOCAL, query=f""" SELECT @@ -213,32 +219,32 @@ def backwards_ops(self) -> Sequence[SqlOperation]: return [ operations.DropTable( storage_set=self.storage_set_key, - table_name=self.str_mv, + table_name=self.new_str_mv, target=OperationTarget.LOCAL, ), operations.DropTable( storage_set=self.storage_set_key, - table_name=self.str_local_table, + table_name=self.new_str_local_table, target=OperationTarget.LOCAL, ), operations.DropTable( storage_set=self.storage_set_key, - table_name=self.str_dist_table, + table_name=self.new_str_dist_table, target=OperationTarget.DISTRIBUTED, ), operations.DropTable( storage_set=self.storage_set_key, - table_name=self.num_mv, + table_name=self.new_num_mv, target=OperationTarget.LOCAL, ), operations.DropTable( storage_set=self.storage_set_key, - table_name=self.num_local_table, + table_name=self.new_num_local_table, target=OperationTarget.LOCAL, ), operations.DropTable( storage_set=self.storage_set_key, - table_name=self.num_dist_table, + table_name=self.new_num_dist_table, target=OperationTarget.DISTRIBUTED, ), ] diff --git a/snuba/web/rpc/trace_item_attribute_values.py b/snuba/web/rpc/trace_item_attribute_values.py index 640937c0ea..bfc9c132c3 100644 --- a/snuba/web/rpc/trace_item_attribute_values.py +++ b/snuba/web/rpc/trace_item_attribute_values.py @@ -110,6 +110,7 @@ def trace_item_attribute_values_query( request=snuba_request, timer=timer, ) + print("res is: ", res) return AttributeValuesResponse( values=[r["attr_value"] for r in res.result.get("data", [])] )