fix(CapMan): Allocation Policies causing potentially timeout errors o…

…n ST (#4403)

snuba/query/allocation_policies/__init__.py

@@ -395,7 +395,10 @@ def is_active(self) -> bool:
 
     @property
     def is_enforced(self) -> bool:
-        return bool(self.get_config_value(IS_ENFORCED))
+        return (
+            bool(self.get_config_value(IS_ENFORCED))
+            and settings.ENFORCE_BYTES_SCANNED_WINDOW_POLICY
+        )
 
     @property
     def max_threads(self) -> int:

snuba/settings/__init__.py

@@ -192,6 +192,7 @@ class RedisClusters(TypedDict):
 BULK_CLICKHOUSE_BUFFER = 10000
 BULK_BINARY_LOAD_CHUNK = 2**22  # 4 MB
 
+
 # Processor/Writer Options
 
 
@@ -249,6 +250,12 @@ class RedisClusters(TypedDict):
 PROJECT_STACKTRACE_BLACKLIST: Set[int] = set()
 PRETTY_FORMAT_EXPRESSIONS = True
 
+# Capacity Management
+# HACK: This is necessary because single tenant does not have snuba-admin deployed / accessible
+# so we can't change policy configs ourselves. This should be removed once we have snuba-admin
+# available for single tenant since we can enable/disable policies at runtime there.
+ENFORCE_BYTES_SCANNED_WINDOW_POLICY = True
+
 # By default, allocation policies won't block requests from going through in a production
 # environment to not cause incidents unnecessarily. If something goes wrong with allocation
 # policy code, the request will still be able to go through (but it will create a dangerous

snuba/settings/settings_test.py

@@ -26,6 +26,8 @@
 # should fail on bad code
 RAISE_ON_ALLOCATION_POLICY_FAILURES = True
 
+ENFORCE_BYTES_SCANNED_WINDOW_POLICY = True
+
 # override replacer threshold to write to redis every time a replacement message is consumed
 REPLACER_PROCESSING_TIMEOUT_THRESHOLD = 0  # ms