ref(ddm): Dual write to Sentry metrics as well as Datadog

snuba/environment.py

@@ -87,6 +87,12 @@ def setup_sentry() -> None:
         release=os.getenv("SNUBA_RELEASE"),
         traces_sample_rate=settings.SENTRY_TRACE_SAMPLE_RATE,
         profiles_sample_rate=settings.SNUBA_PROFILES_SAMPLE_RATE,
+        _experiments={
+            # Turns on the metrics module
+            "enable_metrics": True,
+            # Enables sending of code locations for metrics
+            "metric_code_locations": True,
+        },
     )
 
     from snuba.utils.profiler import run_ondemand_profiler

snuba/utils/metrics/backends/abstract.py

@@ -11,7 +11,11 @@ class MetricsBackend(ABC):
 
     @abstractmethod
     def increment(
-        self, name: str, value: Union[int, float] = 1, tags: Optional[Tags] = None
+        self,
+        name: str,
+        value: Union[int, float] = 1,
+        tags: Optional[Tags] = None,
+        unit: Optional[str] = None,
     ) -> None:
         """
         Increment a counter metric. These increments can also be
@@ -27,7 +31,11 @@ def increment(
 
     @abstractmethod
     def gauge(
-        self, name: str, value: Union[int, float], tags: Optional[Tags] = None
+        self,
+        name: str,
+        value: Union[int, float],
+        tags: Optional[Tags] = None,
+        unit: Optional[str] = None,
     ) -> None:
         """
         Emit a metric that is the authoritative value for a quantity at a point in time
@@ -40,7 +48,11 @@ def gauge(
 
     @abstractmethod
     def timing(
-        self, name: str, value: Union[int, float], tags: Optional[Tags] = None
+        self,
+        name: str,
+        value: Union[int, float],
+        tags: Optional[Tags] = None,
+        unit: Optional[str] = None,
     ) -> None:
         """
         Emit a metric for the timing performance of an operation.
@@ -51,6 +63,23 @@ def timing(
         """
         raise NotImplementedError
 
+    @abstractmethod
+    def distribution(
+        self,
+        name: str,
+        value: Union[int, float],
+        tags: Optional[Tags] = None,
+        unit: Optional[str] = None,
+    ) -> None:
+        """
+        Emit a metric for the performance of an operation.
+
+        Example:
+
+        metrics.distribution("request.size", request_size_in_bytes)
+        """
+        raise NotImplementedError
+
     @abstractmethod
     def events(
         self,

snuba/utils/metrics/backends/datadog.py

@@ -47,7 +47,11 @@ def __normalize_tags(self, tags: Optional[Tags]) -> Optional[Sequence[str]]:
             return [f"{key}:{value.replace('|', '_')}" for key, value in tags.items()]
 
     def increment(
-        self, name: str, value: Union[int, float] = 1, tags: Optional[Tags] = None
+        self,
+        name: str,
+        value: Union[int, float] = 1,
+        tags: Optional[Tags] = None,
+        unit: Optional[str] = None,
     ) -> None:
         self.__client.increment(
             name,
@@ -57,7 +61,11 @@ def increment(
         )
 
     def gauge(
-        self, name: str, value: Union[int, float], tags: Optional[Tags] = None
+        self,
+        name: str,
+        value: Union[int, float],
+        tags: Optional[Tags] = None,
+        unit: Optional[str] = None,
     ) -> None:
         self.__client.gauge(
             name,
@@ -67,7 +75,11 @@ def gauge(
         )
 
     def timing(
-        self, name: str, value: Union[int, float], tags: Optional[Tags] = None
+        self,
+        name: str,
+        value: Union[int, float],
+        tags: Optional[Tags] = None,
+        unit: Optional[str] = None,
     ) -> None:
         self.__client.timing(
             name,
@@ -76,6 +88,20 @@ def timing(
             sample_rate=self.__sample_rates.get(name, 1.0),
         )
 
+    def distribution(
+        self,
+        name: str,
+        value: Union[int, float],
+        tags: Optional[Tags] = None,
+        unit: Optional[str] = None,
+    ) -> None:
+        self.__client.distribution(
+            name,
+            value,
+            tags=self.__normalize_tags(tags),
+            sample_rate=self.__sample_rates.get(name, 1.0),
+        )
+
     def events(
         self,
         title: str,

snuba/utils/metrics/backends/dualwrite.py

@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import random
+
+from snuba import settings, state
+from snuba.utils.metrics.backends.abstract import MetricsBackend
+from snuba.utils.metrics.backends.datadog import DatadogMetricsBackend
+from snuba.utils.metrics.backends.sentry import SentryMetricsBackend
+from snuba.utils.metrics.types import Tags
+
+
+class SentryDatadogMetricsBackend(MetricsBackend):
+    """
+    A metrics backend that records metrics to Sentry and Datadog.
+    """
+
+    def __init__(
+        self, datadog: DatadogMetricsBackend, sentry: SentryMetricsBackend
+    ) -> None:
+        self.datadog = datadog
+        self.sentry = sentry
+
+    def _use_sentry(self) -> bool:
+        if state.get_config("use_sentry_metrics", "0") == "1":
+            return bool(random.random() < settings.DDM_METRICS_SAMPLE_RATE)
+        return False
+
+    def increment(
+        self,
+        name: str,
+        value: int | float = 1,
+        tags: Tags | None = None,
+        unit: str | None = None,
+    ) -> None:
+        self.datadog.increment(name, value, tags, unit)
+        if self._use_sentry():
+            self.sentry.increment(name, value, tags, unit)
+
+    def gauge(
+        self,
+        name: str,
+        value: int | float,
+        tags: Tags | None = None,
+        unit: str | None = None,
+    ) -> None:
+        self.datadog.gauge(name, value, tags, unit)
+        if self._use_sentry():
+            self.sentry.gauge(name, value, tags, unit)
+
+    def timing(
+        self,
+        name: str,
+        value: int | float,
+        tags: Tags | None = None,
+        unit: str | None = None,
+    ) -> None:
+        self.datadog.timing(name, value, tags, unit)
+        if self._use_sentry():
+            self.sentry.timing(name, value, tags, unit)
+
+    def distribution(
+        self,
+        name: str,
+        value: int | float,
+        tags: Tags | None = None,
+        unit: str | None = None,
+    ) -> None:
+        self.datadog.distribution(name, value, tags, unit)
+        if self._use_sentry():
+            self.sentry.distribution(name, value, tags, unit)
+
+    def events(
+        self,
+        title: str,
+        text: str,
+        alert_type: str,
+        priority: str,
+        tags: Tags | None = None,
+    ) -> None:
+        self.datadog.events(title, text, alert_type, priority, tags)

snuba/utils/metrics/backends/dummy.py

@@ -26,7 +26,11 @@ def __validate_tags(self, tags: Tags) -> None:
             assert isinstance(v, str)
 
     def increment(
-        self, name: str, value: Union[int, float] = 1, tags: Optional[Tags] = None
+        self,
+        name: str,
+        value: Union[int, float] = 1,
+        tags: Optional[Tags] = None,
+        unit: Optional[str] = None,
     ) -> None:
         if self.__strict:
             assert isinstance(name, str)
@@ -35,7 +39,11 @@ def increment(
                 self.__validate_tags(tags)
 
     def gauge(
-        self, name: str, value: Union[int, float], tags: Optional[Tags] = None
+        self,
+        name: str,
+        value: Union[int, float],
+        tags: Optional[Tags] = None,
+        unit: Optional[str] = None,
     ) -> None:
         if self.__strict:
             assert isinstance(name, str)
@@ -44,7 +52,24 @@ def gauge(
                 self.__validate_tags(tags)
 
     def timing(
-        self, name: str, value: Union[int, float], tags: Optional[Tags] = None
+        self,
+        name: str,
+        value: Union[int, float],
+        tags: Optional[Tags] = None,
+        unit: Optional[str] = None,
+    ) -> None:
+        if self.__strict:
+            assert isinstance(name, str)
+            assert isinstance(value, (int, float))
+            if tags is not None:
+                self.__validate_tags(tags)
+
+    def distribution(
+        self,
+        name: str,
+        value: Union[int, float],
+        tags: Optional[Tags] = None,
+        unit: Optional[str] = None,
     ) -> None:
         if self.__strict:
             assert isinstance(name, str)

snuba/utils/metrics/backends/sentry.py

@@ -0,0 +1,62 @@
+from __future__ import annotations
+
+from sentry_sdk import metrics
+
+from snuba.utils.metrics.backends.abstract import MetricsBackend
+from snuba.utils.metrics.types import Tags
+
+
+class SentryMetricsBackend(MetricsBackend):
+    """
+    A metrics backend that records metrics to Sentry.
+    """
+
+    def __init__(self) -> None:
+        return None  # Sentry doesn't require any setup
+
+    def increment(
+        self,
+        name: str,
+        value: int | float = 1,
+        tags: Tags | None = None,
+        unit: str | None = None,
+    ) -> None:
+        metrics.incr(name, value, unit or "none", tags)
+
+    def gauge(
+        self,
+        name: str,
+        value: int | float,
+        tags: Tags | None = None,
+        unit: str | None = None,
+    ) -> None:
+        metrics.gauge(name, value, unit or "none", tags)
+
+    def timing(
+        self,
+        name: str,
+        value: int | float,
+        tags: Tags | None = None,
+        unit: str | None = None,
+    ) -> None:
+        # The Sentry SDK has strict typing on the unit, so it doesn't allow passing arbitrary units
+        metrics.timing(name, value, unit or "millisecond", tags)  # type: ignore
+
+    def distribution(
+        self,
+        name: str,
+        value: int | float,
+        tags: Tags | None = None,
+        unit: str | None = None,
+    ) -> None:
+        metrics.distribution(name, value, unit or "none", tags)
+
+    def events(
+        self,
+        title: str,
+        text: str,
+        alert_type: str,
+        priority: str,
+        tags: Tags | None = None,
+    ) -> None:
+        return None  # Sentry doesn't support events