feat(generic-metrics): Retry adding gauges subscriptions topics + con…

…figs (#5892) * add gauges subscriptions * add devserver command
getsentry · May 13, 2024 · 83f3d5a · 83f3d5a
1 parent dab58b1
commit 83f3d5a
Show file tree

Hide file tree

Showing 4 changed files with 35 additions and 5 deletions.
diff --git a/snuba/cli/devserver.py b/snuba/cli/devserver.py
@@ -318,6 +318,19 @@ def devserver(*, bootstrap: bool, workers: bool) -> None:
                             "--schedule-ttl=10",
                         ],
                     ),
+                    (
+                        "subscriptions-scheduler-generic-metrics-gauges",
+                        [
+                            "snuba",
+                            "subscriptions-scheduler",
+                            "--entity=generic_metrics_gauges",
+                            "--consumer-group=snuba-generic-metrics-gauges-subscriptions-schedulers",
+                            "--followed-consumer-group=snuba-generic-metrics-gauges-consumers",
+                            "--auto-offset-reset=latest",
+                            "--log-level=debug",
+                            "--schedule-ttl=10",
+                        ],
+                    ),
                     (
                         "subscriptions-executor-metrics",
                         [

diff --git a/snuba/datasets/configuration/generic_metrics/entities/gauges.yaml b/snuba/datasets/configuration/generic_metrics/entities/gauges.yaml
@@ -26,11 +26,7 @@ schema:
     {
       name: tags.raw_value,
       type: Array,
-      args:
-        {
-          schema_modifiers: [readonly],
-          inner_type: { type: String },
-        },
+      args: { schema_modifiers: [readonly], inner_type: { type: String } },
     },
     {
       name: min,
@@ -259,3 +255,14 @@ validators:
       required_filter_columns: ["org_id", "project_id"]
 required_time_column: timestamp
 partition_key_column_name: org_id
+subscription_processors:
+  - processor: AddColumnCondition
+    args:
+      extra_condition_data_key: organization
+      extra_condition_column: org_id
+subscription_validators:
+  - validator: AggregationValidator
+    args:
+      max_allowed_aggregations: 3
+      disallowed_aggregations: [having, orderby]
+      required_time_column: timestamp
diff --git a/snuba/datasets/configuration/generic_metrics/storages/gauges_bucket.yaml b/snuba/datasets/configuration/generic_metrics/storages/gauges_bucket.yaml
@@ -78,4 +78,10 @@ stream_loader:
     args:
       header_key: metric_type
       header_value: g
+  commit_log_topic: snuba-generic-metrics-gauges-commit-log
+  subscription_scheduler_mode: global
+  subscription_synchronization_timestamp: orig_message_ts
+  subscription_scheduled_topic: scheduled-subscriptions-generic-metrics-gauges
+  subscription_result_topic: generic-metrics-subscription-results
+  subscription_delay_seconds: 60
   dlq_topic: snuba-dead-letter-generic-metrics
diff --git a/snuba/utils/streams/topics.py b/snuba/utils/streams/topics.py
@@ -29,6 +29,9 @@ class Topic(Enum):
     SUBSCRIPTION_SCHEDULED_GENERIC_METRICS_COUNTERS = (
         "scheduled-subscriptions-generic-metrics-counters"
     )
+    SUBSCRIPTION_SCHEDULED_GENERIC_METRICS_GAUGES = (
+        "scheduled-subscriptions-generic-metrics-gauges"
+    )
 
     SUBSCRIPTION_RESULTS_EVENTS = "events-subscription-results"
     SUBSCRIPTION_RESULTS_TRANSACTIONS = "transactions-subscription-results"
@@ -46,6 +49,7 @@ class Topic(Enum):
         "snuba-generic-metrics-distributions-commit-log"
     )
     GENERIC_METRICS_COUNTERS_COMMIT_LOG = "snuba-generic-metrics-counters-commit-log"
+    GENERIC_METRICS_GAUGES_COMMIT_LOG = "snuba-generic-metrics-gauges-commit-log"
     GENERIC_EVENTS = "generic-events"
     GENERIC_EVENTS_COMMIT_LOG = "snuba-generic-events-commit-log"
     GROUP_ATTRIBUTES = "group-attributes"