From 6802ffa6d4a929e0bb7c0b5cac31472a13bf1ba6 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Fri, 13 Sep 2024 14:53:22 -0400 Subject: [PATCH 1/9] feat(eap): implement backend for "what tag values" RPC --- ...{tags_list.py => trace_item_attributes.py} | 2 +- snuba/web/rpc/trace_item_values.py | 111 +++++++++++++++ snuba/web/views.py | 10 +- ..._list.py => test_trace_item_attributes.py} | 8 +- tests/web/rpc/test_trace_item_values.py | 127 ++++++++++++++++++ 5 files changed, 251 insertions(+), 7 deletions(-) rename snuba/web/rpc/{tags_list.py => trace_item_attributes.py} (98%) create mode 100644 snuba/web/rpc/trace_item_values.py rename tests/web/rpc/{test_tags_list.py => test_trace_item_attributes.py} (97%) create mode 100644 tests/web/rpc/test_trace_item_values.py diff --git a/snuba/web/rpc/tags_list.py b/snuba/web/rpc/trace_item_attributes.py similarity index 98% rename from snuba/web/rpc/tags_list.py rename to snuba/web/rpc/trace_item_attributes.py index e0dbb81421..a05f89c623 100644 --- a/snuba/web/rpc/tags_list.py +++ b/snuba/web/rpc/trace_item_attributes.py @@ -15,7 +15,7 @@ from snuba.web.rpc.exceptions import BadSnubaRPCRequestException -def tags_list_query( +def trace_items_attributes_query( request: TraceItemAttributesRequest, _timer: Optional[Timer] = None ) -> TraceItemAttributesResponse: if request.type == AttributeKey.Type.TYPE_STRING: diff --git a/snuba/web/rpc/trace_item_values.py b/snuba/web/rpc/trace_item_values.py new file mode 100644 index 0000000000..aa299114d1 --- /dev/null +++ b/snuba/web/rpc/trace_item_values.py @@ -0,0 +1,111 @@ +import uuid +from datetime import datetime + +from google.protobuf.json_format import MessageToDict +from sentry_protos.snuba.v1alpha.endpoint_tags_list_pb2 import ( + AttributeValuesRequest, + AttributeValuesResponse, +) + +from snuba.attribution.appid import AppID +from snuba.attribution.attribution_info import AttributionInfo +from snuba.datasets.entities.entity_key import EntityKey +from snuba.datasets.entities.factory import get_entity +from snuba.datasets.pluggable_dataset import PluggableDataset +from snuba.query import OrderBy, OrderByDirection, SelectedExpression +from snuba.query.data_source.simple import Entity +from snuba.query.dsl import Functions as f +from snuba.query.dsl import column, literal, literals_array +from snuba.query.logical import Query +from snuba.query.query_settings import HTTPQuerySettings +from snuba.request import Request as SnubaRequest +from snuba.utils.metrics.timer import Timer +from snuba.web.query import run_query +from snuba.web.rpc.common import base_conditions_and, treeify_or_and_conditions +from snuba.web.rpc.exceptions import BadSnubaRPCRequestException + + +def _build_query(request: AttributeValuesRequest) -> Query: + if request.limit > 1000: + raise BadSnubaRPCRequestException("Limit can be at most 1000") + + entity = Entity( + key=EntityKey("spans_str_attrs"), + schema=get_entity(EntityKey("spans_str_attrs")).get_data_model(), + sample=None, + ) + + # this table stores timestamp as toStartOfDay(x) in UTC, so if you request 4PM - 8PM on a specific day, nada + start_timestamp = datetime.utcfromtimestamp(request.meta.start_timestamp.seconds) + end_timestamp = datetime.utcfromtimestamp(request.meta.end_timestamp.seconds) + if start_timestamp.day == end_timestamp.day: + start_timestamp = start_timestamp.replace( + day=start_timestamp.day - 1, hour=0, minute=0, second=0, microsecond=0 + ) + end_timestamp = end_timestamp.replace(day=end_timestamp.day + 1) + request.meta.start_timestamp.seconds = int(start_timestamp.timestamp()) + request.meta.end_timestamp.seconds = int(end_timestamp.timestamp()) + + res = Query( + from_clause=entity, + selected_columns=[ + SelectedExpression( + name="attr_value", expression=column("attr_value", alias="attr_value") + ), + ], + condition=base_conditions_and( + request.meta, + f.equals(column("attr_key"), literal(request.name)), + # multiSearchAny has special treatment with bloom filters + # https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#functions-support + f.multiSearchAny( + column("attr_value"), + literals_array(None, [literal(request.value_substring_match)]), + ), + ), + groupby=[column("attr_value", alias="attr_value")], + order_by=[ + OrderBy(direction=OrderByDirection.ASC, expression=column("attr_value")) + ], + limit=request.limit, + offset=request.offset, + ) + treeify_or_and_conditions(res) + return res + + +def _build_snuba_request( + request: AttributeValuesRequest, +) -> SnubaRequest: + return SnubaRequest( + id=str(uuid.uuid4()), + original_body=MessageToDict(request), + query=_build_query(request), + query_settings=HTTPQuerySettings(), + attribution_info=AttributionInfo( + referrer=request.meta.referrer, + team="eap", + feature="eap", + tenant_ids={ + "organization_id": request.meta.organization_id, + "referrer": request.meta.referrer, + }, + app_id=AppID("eap"), + parent_api="trace_item_values", + ), + ) + + +def trace_item_values_query( + request: AttributeValuesRequest, timer: Timer | None = None +) -> AttributeValuesResponse: + timer = timer or Timer("trace_item_values") + snuba_request = _build_snuba_request(request) + res = run_query( + dataset=PluggableDataset(name="eap", all_entities=[]), + request=snuba_request, + timer=timer, + ) + return AttributeValuesResponse( + values=[r["attr_value"] for r in res.result.get("data", [])] + ) diff --git a/snuba/web/views.py b/snuba/web/views.py index 720f700123..ca8f0d49bc 100644 --- a/snuba/web/views.py +++ b/snuba/web/views.py @@ -41,6 +41,7 @@ ) from sentry_protos.snuba.v1alpha.endpoint_span_samples_pb2 import SpanSamplesRequest from sentry_protos.snuba.v1alpha.endpoint_tags_list_pb2 import ( + AttributeValuesRequest, TraceItemAttributesRequest, ) from werkzeug import Response as WerkzeugResponse @@ -85,8 +86,9 @@ from snuba.web.query import parse_and_run_query from snuba.web.rpc.exceptions import BadSnubaRPCRequestException from snuba.web.rpc.span_samples import span_samples_query as span_samples_query -from snuba.web.rpc.tags_list import tags_list_query from snuba.web.rpc.timeseries import timeseries_query as timeseries_query +from snuba.web.rpc.trace_item_attributes import trace_items_attributes_query +from snuba.web.rpc.trace_item_values import trace_item_values_query from snuba.writer import BatchWriterEncoderWrapper, WriterTableRow logger = logging.getLogger("snuba.api") @@ -290,7 +292,11 @@ def rpc(*, name: str, timer: Timer) -> Response: ] = { "AggregateBucketRequest": (timeseries_query, AggregateBucketRequest), "SpanSamplesRequest": (span_samples_query, SpanSamplesRequest), - "TraceItemAttributesRequest": (tags_list_query, TraceItemAttributesRequest), + "TraceItemAttributesRequest": ( + trace_items_attributes_query, + TraceItemAttributesRequest, + ), + "AttributeValuesRequest": (trace_item_values_query, AttributeValuesRequest), } try: endpoint, req_class = rpcs[name] diff --git a/tests/web/rpc/test_tags_list.py b/tests/web/rpc/test_trace_item_attributes.py similarity index 97% rename from tests/web/rpc/test_tags_list.py rename to tests/web/rpc/test_trace_item_attributes.py index ff353fb9af..cae5e67b15 100644 --- a/tests/web/rpc/test_tags_list.py +++ b/tests/web/rpc/test_trace_item_attributes.py @@ -13,7 +13,7 @@ from snuba.datasets.storages.factory import get_storage from snuba.datasets.storages.storage_key import StorageKey -from snuba.web.rpc.tags_list import tags_list_query +from snuba.web.rpc.trace_item_attributes import trace_items_attributes_query from tests.base import BaseApiTest from tests.helpers import write_raw_unprocessed_events @@ -135,7 +135,7 @@ def test_simple_case_str(self, setup_teardown: Any) -> None: offset=0, type=AttributeKey.Type.TYPE_STRING, ) - response = tags_list_query(message) + response = trace_items_attributes_query(message) assert response.tags == [ TraceItemAttributesResponse.Tag( name=f"a_tag_{i:03}", type=AttributeKey.Type.TYPE_STRING @@ -175,7 +175,7 @@ def test_simple_case_float(self, setup_teardown: Any) -> None: offset=0, type=AttributeKey.Type.TYPE_FLOAT, ) - response = tags_list_query(message) + response = trace_items_attributes_query(message) assert response.tags == [ TraceItemAttributesResponse.Tag( name=f"b_measurement_{i:03}", type=AttributeKey.Type.TYPE_FLOAT @@ -215,7 +215,7 @@ def test_with_offset(self, setup_teardown: Any) -> None: offset=10, type=AttributeKey.Type.TYPE_FLOAT, ) - response = tags_list_query(message) + response = trace_items_attributes_query(message) assert response.tags == [ TraceItemAttributesResponse.Tag( name="b_measurement_010", type=AttributeKey.Type.TYPE_FLOAT diff --git a/tests/web/rpc/test_trace_item_values.py b/tests/web/rpc/test_trace_item_values.py new file mode 100644 index 0000000000..eb0bcebd7f --- /dev/null +++ b/tests/web/rpc/test_trace_item_values.py @@ -0,0 +1,127 @@ +import uuid +from datetime import UTC, datetime, timedelta +from typing import Any, Mapping + +import pytest +from google.protobuf.timestamp_pb2 import Timestamp +from sentry_protos.snuba.v1alpha.endpoint_tags_list_pb2 import AttributeValuesRequest +from sentry_protos.snuba.v1alpha.request_common_pb2 import RequestMeta + +from snuba.datasets.storages.factory import get_storage +from snuba.datasets.storages.storage_key import StorageKey +from snuba.web.rpc.trace_item_values import trace_item_values_query +from tests.base import BaseApiTest +from tests.helpers import write_raw_unprocessed_events + +BASE_TIME = datetime.utcnow().replace(minute=0, second=0, microsecond=0) - timedelta( + minutes=180 +) +COMMON_META = RequestMeta( + project_ids=[1, 2, 3], + organization_id=1, + cogs_category="something", + referrer="something", + start_timestamp=Timestamp( + seconds=int( + datetime( + year=BASE_TIME.year, + month=BASE_TIME.month, + day=BASE_TIME.day, + tzinfo=UTC, + ).timestamp() + ) + ), + end_timestamp=Timestamp( + seconds=int( + datetime( + year=BASE_TIME.year, + month=BASE_TIME.month, + day=BASE_TIME.day + 1, + tzinfo=UTC, + ).timestamp() + ) + ), +) + + +def gen_message(tags) -> Mapping[str, Any]: + return { + "description": "/api/0/relays/projectconfigs/", + "duration_ms": 152, + "event_id": "d826225de75d42d6b2f01b957d51f18f", + "exclusive_time_ms": 0.228, + "is_segment": True, + "data": {}, + "measurements": {}, + "organization_id": 1, + "origin": "auto.http.django", + "project_id": 1, + "received": 1721319572.877828, + "retention_days": 90, + "segment_id": "8873a98879faf06d", + "sentry_tags": {}, + "span_id": uuid.uuid4().hex, + "tags": tags, + "trace_id": uuid.uuid4().hex, + "start_timestamp_ms": int(BASE_TIME.timestamp() * 1000), + "start_timestamp_precise": BASE_TIME.timestamp(), + "end_timestamp_precise": BASE_TIME.timestamp() + 1, + } + + +@pytest.fixture(autouse=True) +def setup_teardown(clickhouse_db: None, redis_db: None) -> None: + spans_storage = get_storage(StorageKey("eap_spans")) + messages = [ + gen_message({"tag1": "herp", "tag2": "herp"}), + gen_message({"tag1": "herpderp", "tag2": "herp"}), + gen_message({"tag1": "durp", "tag3": "herp"}), + gen_message({"tag1": "blah", "tag2": "herp"}), + gen_message({"tag1": "derpderp", "tag2": "derp"}), + gen_message({"tag2": "hehe"}), + gen_message({"tag1": "some_last_value"}), + ] + write_raw_unprocessed_events(spans_storage, messages) # type: ignore + + +@pytest.mark.clickhouse_db +@pytest.mark.redis_db +class TestTraceItemAttributes(BaseApiTest): + def test_basic(self) -> None: + ts = Timestamp() + ts.GetCurrentTime() + message = AttributeValuesRequest( + meta=COMMON_META, + name="tag1", + limit=10, + offset=20, + ) + response = self.app.post( + "/rpc/AttributeValuesRequest", data=message.SerializeToString() + ) + assert response.status_code == 200 + + def test_simple_case(self, setup_teardown: Any) -> None: + message = AttributeValuesRequest(meta=COMMON_META, limit=5, name="tag1") + response = trace_item_values_query(message) + assert response.values == ["blah", "derpderp", "durp", "herp", "herpderp"] + + def test_offset(self, setup_teardown: Any) -> None: + message = AttributeValuesRequest( + meta=COMMON_META, limit=5, offset=1, name="tag1" + ) + response = trace_item_values_query(message) + assert response.values == [ + "derpderp", + "durp", + "herp", + "herpderp", + "some_last_value", + ] + + def test_with_value_filter(self, setup_teardown: Any) -> None: + message = AttributeValuesRequest( + meta=COMMON_META, limit=5, name="tag1", value_substring_match="erp" + ) + response = trace_item_values_query(message) + assert response.values == ["derpderp", "herp", "herpderp"] From 83f33e4602ce513605fb8868e3deee6f86120b27 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Fri, 13 Sep 2024 14:58:09 -0400 Subject: [PATCH 2/9] mypy --- tests/web/rpc/test_trace_item_values.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/web/rpc/test_trace_item_values.py b/tests/web/rpc/test_trace_item_values.py index eb0bcebd7f..234c77439c 100644 --- a/tests/web/rpc/test_trace_item_values.py +++ b/tests/web/rpc/test_trace_item_values.py @@ -44,7 +44,7 @@ ) -def gen_message(tags) -> Mapping[str, Any]: +def gen_message(tags: Mapping[str, str]) -> Mapping[str, Any]: return { "description": "/api/0/relays/projectconfigs/", "duration_ms": 152, From 9dbbe20295b665bdd1e5dffc95f23994b19b3b47 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Fri, 13 Sep 2024 15:02:11 -0400 Subject: [PATCH 3/9] doc --- snuba/web/rpc/trace_item_values.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/snuba/web/rpc/trace_item_values.py b/snuba/web/rpc/trace_item_values.py index aa299114d1..38273c24f4 100644 --- a/snuba/web/rpc/trace_item_values.py +++ b/snuba/web/rpc/trace_item_values.py @@ -56,7 +56,7 @@ def _build_query(request: AttributeValuesRequest) -> Query: condition=base_conditions_and( request.meta, f.equals(column("attr_key"), literal(request.name)), - # multiSearchAny has special treatment with bloom filters + # multiSearchAny has special treatment with ngram bloom filters # https://clickhouse.com/docs/en/engines/table-engines/mergetree-family/mergetree#functions-support f.multiSearchAny( column("attr_value"), From 28be6b1fc695addcb8b0b517c26ca244295f3790 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Tue, 17 Sep 2024 11:32:44 -0400 Subject: [PATCH 4/9] fix: perf --- snuba/web/rpc/trace_item_values.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/snuba/web/rpc/trace_item_values.py b/snuba/web/rpc/trace_item_values.py index 38273c24f4..539000cdc9 100644 --- a/snuba/web/rpc/trace_item_values.py +++ b/snuba/web/rpc/trace_item_values.py @@ -65,7 +65,11 @@ def _build_query(request: AttributeValuesRequest) -> Query: ), groupby=[column("attr_value", alias="attr_value")], order_by=[ - OrderBy(direction=OrderByDirection.ASC, expression=column("attr_value")) + OrderBy( + direction=OrderByDirection.ASC, expression=column("organization_id") + ), + OrderBy(direction=OrderByDirection.ASC, expression=column("attr_key")), + OrderBy(direction=OrderByDirection.ASC, expression=column("attr_value")), ], limit=request.limit, offset=request.offset, From ba232009dc809ec7b437b327cd5458b3c4764ca5 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Tue, 17 Sep 2024 12:17:29 -0400 Subject: [PATCH 5/9] fix test --- snuba/web/rpc/trace_item_values.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/snuba/web/rpc/trace_item_values.py b/snuba/web/rpc/trace_item_values.py index 539000cdc9..2e0c379a76 100644 --- a/snuba/web/rpc/trace_item_values.py +++ b/snuba/web/rpc/trace_item_values.py @@ -63,7 +63,11 @@ def _build_query(request: AttributeValuesRequest) -> Query: literals_array(None, [literal(request.value_substring_match)]), ), ), - groupby=[column("attr_value", alias="attr_value")], + groupby=[ + column("organization_id", alias="organization_id"), + column("attr_key", alias="attr_key"), + column("attr_value", alias="attr_value"), + ], order_by=[ OrderBy( direction=OrderByDirection.ASC, expression=column("organization_id") From 63a6672da73857315738f63c5542f73fa7601f55 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Tue, 17 Sep 2024 13:03:51 -0400 Subject: [PATCH 6/9] Move RPC list into rpc folder --- snuba/web/rpc/__init__.py | 29 +++++++++++++++++++++++++++++ snuba/web/views.py | 27 ++------------------------- 2 files changed, 31 insertions(+), 25 deletions(-) create mode 100644 snuba/web/rpc/__init__.py diff --git a/snuba/web/rpc/__init__.py b/snuba/web/rpc/__init__.py new file mode 100644 index 0000000000..e2ccd8754d --- /dev/null +++ b/snuba/web/rpc/__init__.py @@ -0,0 +1,29 @@ +from typing import Any, Callable, Mapping, Tuple + +from google.protobuf.message import Message as ProtobufMessage +from sentry_protos.snuba.v1alpha.endpoint_aggregate_bucket_pb2 import ( + AggregateBucketRequest, +) +from sentry_protos.snuba.v1alpha.endpoint_span_samples_pb2 import SpanSamplesRequest +from sentry_protos.snuba.v1alpha.endpoint_tags_list_pb2 import ( + AttributeValuesRequest, + TraceItemAttributesRequest, +) + +from snuba.utils.metrics.timer import Timer +from snuba.web.rpc.span_samples import span_samples_query +from snuba.web.rpc.timeseries import timeseries_query +from snuba.web.rpc.trace_item_attributes import trace_items_attributes_query +from snuba.web.rpc.trace_item_values import trace_item_values_query + +ALL_RPCS: Mapping[ + str, Tuple[Callable[[Any, Timer], ProtobufMessage], type[ProtobufMessage]] +] = { + "AggregateBucketRequest": (timeseries_query, AggregateBucketRequest), + "SpanSamplesRequest": (span_samples_query, SpanSamplesRequest), + "TraceItemAttributesRequest": ( + trace_items_attributes_query, + TraceItemAttributesRequest, + ), + "AttributeValuesRequest": (trace_item_values_query, AttributeValuesRequest), +} diff --git a/snuba/web/views.py b/snuba/web/views.py index ca8f0d49bc..1861f28937 100644 --- a/snuba/web/views.py +++ b/snuba/web/views.py @@ -35,15 +35,6 @@ render_template, ) from flask import request as http_request -from google.protobuf.message import Message as ProtobufMessage -from sentry_protos.snuba.v1alpha.endpoint_aggregate_bucket_pb2 import ( - AggregateBucketRequest, -) -from sentry_protos.snuba.v1alpha.endpoint_span_samples_pb2 import SpanSamplesRequest -from sentry_protos.snuba.v1alpha.endpoint_tags_list_pb2 import ( - AttributeValuesRequest, - TraceItemAttributesRequest, -) from werkzeug import Response as WerkzeugResponse from werkzeug.exceptions import InternalServerError @@ -84,11 +75,8 @@ from snuba.web.converters import DatasetConverter, EntityConverter, StorageConverter from snuba.web.delete_query import DeletesNotEnabledError, delete_from_storage from snuba.web.query import parse_and_run_query +from snuba.web.rpc import ALL_RPCS from snuba.web.rpc.exceptions import BadSnubaRPCRequestException -from snuba.web.rpc.span_samples import span_samples_query as span_samples_query -from snuba.web.rpc.timeseries import timeseries_query as timeseries_query -from snuba.web.rpc.trace_item_attributes import trace_items_attributes_query -from snuba.web.rpc.trace_item_values import trace_item_values_query from snuba.writer import BatchWriterEncoderWrapper, WriterTableRow logger = logging.getLogger("snuba.api") @@ -287,19 +275,8 @@ def unqualified_query_view(*, timer: Timer) -> Union[Response, str, WerkzeugResp @application.route("/rpc/", methods=["POST"]) @util.time_request("timeseries") def rpc(*, name: str, timer: Timer) -> Response: - rpcs: Mapping[ - str, Tuple[Callable[[Any, Timer], ProtobufMessage], type[ProtobufMessage]] - ] = { - "AggregateBucketRequest": (timeseries_query, AggregateBucketRequest), - "SpanSamplesRequest": (span_samples_query, SpanSamplesRequest), - "TraceItemAttributesRequest": ( - trace_items_attributes_query, - TraceItemAttributesRequest, - ), - "AttributeValuesRequest": (trace_item_values_query, AttributeValuesRequest), - } try: - endpoint, req_class = rpcs[name] + endpoint, req_class = ALL_RPCS[name] req = req_class() req.ParseFromString(http_request.data) From 975f103d25f6e71bdfc16f9106292f0c96bb57bb Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Tue, 17 Sep 2024 13:12:16 -0400 Subject: [PATCH 7/9] Improve performance --- snuba/web/rpc/trace_item_values.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/snuba/web/rpc/trace_item_values.py b/snuba/web/rpc/trace_item_values.py index 2e0c379a76..4ed5337b7e 100644 --- a/snuba/web/rpc/trace_item_values.py +++ b/snuba/web/rpc/trace_item_values.py @@ -50,7 +50,8 @@ def _build_query(request: AttributeValuesRequest) -> Query: from_clause=entity, selected_columns=[ SelectedExpression( - name="attr_value", expression=column("attr_value", alias="attr_value") + name="attr_value", + expression=f.distinct(column("attr_value", alias="attr_value")), ), ], condition=base_conditions_and( @@ -63,11 +64,6 @@ def _build_query(request: AttributeValuesRequest) -> Query: literals_array(None, [literal(request.value_substring_match)]), ), ), - groupby=[ - column("organization_id", alias="organization_id"), - column("attr_key", alias="attr_key"), - column("attr_value", alias="attr_value"), - ], order_by=[ OrderBy( direction=OrderByDirection.ASC, expression=column("organization_id") From 26d5e85abeda15187850896e1adba6f0c82eb7d7 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Tue, 17 Sep 2024 13:25:35 -0400 Subject: [PATCH 8/9] chore: a few renames --- snuba/web/rpc/__init__.py | 11 +++++++---- ...tem_attributes.py => trace_item_attribute_list.py} | 2 +- ..._item_values.py => trace_item_attribute_values.py} | 2 +- ...ttributes.py => test_trace_item_attribute_list.py} | 8 ++++---- ..._values.py => test_trace_item_attribute_values.py} | 8 ++++---- 5 files changed, 17 insertions(+), 14 deletions(-) rename snuba/web/rpc/{trace_item_attributes.py => trace_item_attribute_list.py} (98%) rename snuba/web/rpc/{trace_item_values.py => trace_item_attribute_values.py} (99%) rename tests/web/rpc/{test_trace_item_attributes.py => test_trace_item_attribute_list.py} (96%) rename tests/web/rpc/{test_trace_item_values.py => test_trace_item_attribute_values.py} (93%) diff --git a/snuba/web/rpc/__init__.py b/snuba/web/rpc/__init__.py index e2ccd8754d..6034257542 100644 --- a/snuba/web/rpc/__init__.py +++ b/snuba/web/rpc/__init__.py @@ -13,8 +13,8 @@ from snuba.utils.metrics.timer import Timer from snuba.web.rpc.span_samples import span_samples_query from snuba.web.rpc.timeseries import timeseries_query -from snuba.web.rpc.trace_item_attributes import trace_items_attributes_query -from snuba.web.rpc.trace_item_values import trace_item_values_query +from snuba.web.rpc.trace_item_attribute_list import trace_item_attribute_list_query +from snuba.web.rpc.trace_item_attribute_values import trace_item_attribute_values_query ALL_RPCS: Mapping[ str, Tuple[Callable[[Any, Timer], ProtobufMessage], type[ProtobufMessage]] @@ -22,8 +22,11 @@ "AggregateBucketRequest": (timeseries_query, AggregateBucketRequest), "SpanSamplesRequest": (span_samples_query, SpanSamplesRequest), "TraceItemAttributesRequest": ( - trace_items_attributes_query, + trace_item_attribute_list_query, TraceItemAttributesRequest, ), - "AttributeValuesRequest": (trace_item_values_query, AttributeValuesRequest), + "AttributeValuesRequest": ( + trace_item_attribute_values_query, + AttributeValuesRequest, + ), } diff --git a/snuba/web/rpc/trace_item_attributes.py b/snuba/web/rpc/trace_item_attribute_list.py similarity index 98% rename from snuba/web/rpc/trace_item_attributes.py rename to snuba/web/rpc/trace_item_attribute_list.py index a05f89c623..f0a159bdf9 100644 --- a/snuba/web/rpc/trace_item_attributes.py +++ b/snuba/web/rpc/trace_item_attribute_list.py @@ -15,7 +15,7 @@ from snuba.web.rpc.exceptions import BadSnubaRPCRequestException -def trace_items_attributes_query( +def trace_item_attribute_list_query( request: TraceItemAttributesRequest, _timer: Optional[Timer] = None ) -> TraceItemAttributesResponse: if request.type == AttributeKey.Type.TYPE_STRING: diff --git a/snuba/web/rpc/trace_item_values.py b/snuba/web/rpc/trace_item_attribute_values.py similarity index 99% rename from snuba/web/rpc/trace_item_values.py rename to snuba/web/rpc/trace_item_attribute_values.py index 4ed5337b7e..640937c0ea 100644 --- a/snuba/web/rpc/trace_item_values.py +++ b/snuba/web/rpc/trace_item_attribute_values.py @@ -100,7 +100,7 @@ def _build_snuba_request( ) -def trace_item_values_query( +def trace_item_attribute_values_query( request: AttributeValuesRequest, timer: Timer | None = None ) -> AttributeValuesResponse: timer = timer or Timer("trace_item_values") diff --git a/tests/web/rpc/test_trace_item_attributes.py b/tests/web/rpc/test_trace_item_attribute_list.py similarity index 96% rename from tests/web/rpc/test_trace_item_attributes.py rename to tests/web/rpc/test_trace_item_attribute_list.py index cae5e67b15..ecb0846e70 100644 --- a/tests/web/rpc/test_trace_item_attributes.py +++ b/tests/web/rpc/test_trace_item_attribute_list.py @@ -13,7 +13,7 @@ from snuba.datasets.storages.factory import get_storage from snuba.datasets.storages.storage_key import StorageKey -from snuba.web.rpc.trace_item_attributes import trace_items_attributes_query +from snuba.web.rpc.trace_item_attribute_list import trace_item_attribute_list_query from tests.base import BaseApiTest from tests.helpers import write_raw_unprocessed_events @@ -135,7 +135,7 @@ def test_simple_case_str(self, setup_teardown: Any) -> None: offset=0, type=AttributeKey.Type.TYPE_STRING, ) - response = trace_items_attributes_query(message) + response = trace_item_attribute_list_query(message) assert response.tags == [ TraceItemAttributesResponse.Tag( name=f"a_tag_{i:03}", type=AttributeKey.Type.TYPE_STRING @@ -175,7 +175,7 @@ def test_simple_case_float(self, setup_teardown: Any) -> None: offset=0, type=AttributeKey.Type.TYPE_FLOAT, ) - response = trace_items_attributes_query(message) + response = trace_item_attribute_list_query(message) assert response.tags == [ TraceItemAttributesResponse.Tag( name=f"b_measurement_{i:03}", type=AttributeKey.Type.TYPE_FLOAT @@ -215,7 +215,7 @@ def test_with_offset(self, setup_teardown: Any) -> None: offset=10, type=AttributeKey.Type.TYPE_FLOAT, ) - response = trace_items_attributes_query(message) + response = trace_item_attribute_list_query(message) assert response.tags == [ TraceItemAttributesResponse.Tag( name="b_measurement_010", type=AttributeKey.Type.TYPE_FLOAT diff --git a/tests/web/rpc/test_trace_item_values.py b/tests/web/rpc/test_trace_item_attribute_values.py similarity index 93% rename from tests/web/rpc/test_trace_item_values.py rename to tests/web/rpc/test_trace_item_attribute_values.py index 234c77439c..3f400219be 100644 --- a/tests/web/rpc/test_trace_item_values.py +++ b/tests/web/rpc/test_trace_item_attribute_values.py @@ -9,7 +9,7 @@ from snuba.datasets.storages.factory import get_storage from snuba.datasets.storages.storage_key import StorageKey -from snuba.web.rpc.trace_item_values import trace_item_values_query +from snuba.web.rpc.trace_item_attribute_values import trace_item_attribute_values_query from tests.base import BaseApiTest from tests.helpers import write_raw_unprocessed_events @@ -103,14 +103,14 @@ def test_basic(self) -> None: def test_simple_case(self, setup_teardown: Any) -> None: message = AttributeValuesRequest(meta=COMMON_META, limit=5, name="tag1") - response = trace_item_values_query(message) + response = trace_item_attribute_values_query(message) assert response.values == ["blah", "derpderp", "durp", "herp", "herpderp"] def test_offset(self, setup_teardown: Any) -> None: message = AttributeValuesRequest( meta=COMMON_META, limit=5, offset=1, name="tag1" ) - response = trace_item_values_query(message) + response = trace_item_attribute_values_query(message) assert response.values == [ "derpderp", "durp", @@ -123,5 +123,5 @@ def test_with_value_filter(self, setup_teardown: Any) -> None: message = AttributeValuesRequest( meta=COMMON_META, limit=5, name="tag1", value_substring_match="erp" ) - response = trace_item_values_query(message) + response = trace_item_attribute_values_query(message) assert response.values == ["derpderp", "herp", "herpderp"] From 97dfc6050dc93dcdc78b8f76c41ba1be69eb5d17 Mon Sep 17 00:00:00 2001 From: Colin Chartier Date: Tue, 17 Sep 2024 13:27:16 -0400 Subject: [PATCH 9/9] chore: make trace item attribute list permissive like values is --- snuba/web/rpc/trace_item_attribute_list.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/snuba/web/rpc/trace_item_attribute_list.py b/snuba/web/rpc/trace_item_attribute_list.py index f0a159bdf9..4ba75235a0 100644 --- a/snuba/web/rpc/trace_item_attribute_list.py +++ b/snuba/web/rpc/trace_item_attribute_list.py @@ -31,11 +31,16 @@ def trace_item_attribute_list_query( if request.limit > 1000: raise BadSnubaRPCRequestException("Limit can be at most 1000") + # this table stores timestamp as toStartOfDay(x) in UTC, so if you request 4PM - 8PM on a specific day, nada start_timestamp = datetime.utcfromtimestamp(request.meta.start_timestamp.seconds) - if start_timestamp.day >= datetime.utcnow().day and start_timestamp.hour != 0: - raise BadSnubaRPCRequestException( - "Tags' timestamps are stored per-day, you probably want to set start_timestamp to UTC 00:00 today or a time yesterday." + end_timestamp = datetime.utcfromtimestamp(request.meta.end_timestamp.seconds) + if start_timestamp.day == end_timestamp.day: + start_timestamp = start_timestamp.replace( + day=start_timestamp.day - 1, hour=0, minute=0, second=0, microsecond=0 ) + end_timestamp = end_timestamp.replace(day=end_timestamp.day + 1) + request.meta.start_timestamp.seconds = int(start_timestamp.timestamp()) + request.meta.end_timestamp.seconds = int(end_timestamp.timestamp()) query = f""" SELECT DISTINCT attr_key, timestamp