From 2bb6fc5fb3b489bc7a619a9cab39bf15d39b6b07 Mon Sep 17 00:00:00 2001 From: Lyn Nagara Date: Wed, 21 Jun 2023 15:54:51 -0700 Subject: [PATCH] feat: Occasionally log errors if no schema found (#4392) We use the validation errors in prod (especially the attached message.txt) to iterate on the schema. This is kind of a hack to get hold of example messages before a schema is defined to help speed up the process of creating a schema for topics where we don't have them yet. --- snuba/consumers/consumer.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/snuba/consumers/consumer.py b/snuba/consumers/consumer.py index deb75dd3f0..e8137bfb7b 100644 --- a/snuba/consumers/consumer.py +++ b/snuba/consumers/consumer.py @@ -55,7 +55,7 @@ from snuba import environment, state from snuba.clickhouse.http import JSONRow, JSONRowEncoder, ValuesRowEncoder -from snuba.consumers.schemas import get_json_codec +from snuba.consumers.schemas import _NOOP_CODEC, get_json_codec from snuba.consumers.types import KafkaMessageMetadata from snuba.datasets.storage import WritableTableStorage from snuba.datasets.storages.factory import get_writable_storage @@ -553,6 +553,10 @@ def process_message( scope.set_tag("snuba_logical_topic", snuba_logical_topic.name) try: + # Occasionally log errors if no validator is configured + if codec == _NOOP_CODEC: + raise Exception("No validator configured for topic") + codec.validate(decoded) except Exception as err: local_metrics.increment(