(dynamic alert thresholds) Force recompilation of Stumpy library duri…

…ng bootup (#1140)

Use mode preloading mechanism to force numba compilation for Stumpy
library

docker-compose.yml

@@ -51,6 +51,7 @@ services:
  - DEV=1
  - SEVERITY_ENABLED=1
  - GROUPING_ENABLED=1
+ - ANOMALY_DETECTION_ENABLED=1
  - CELERY_WORKER_ENABLE=true
  - PORT=9091
  - SMOKE_TEST=1

src/seer/anomaly_detection/anomaly_detection.py

@@ -1,7 +1,9 @@
 import logging
 from typing import List, Tuple
 
+import numpy as np
 import sentry_sdk
+import stumpy # type: ignore # mypy throws "missing library stubs"
 from pydantic import BaseModel
 
 from seer.anomaly_detection.accessors import AlertDataAccessor
@@ -27,6 +29,25 @@
 
 
 class AnomalyDetection(BaseModel):
+ @sentry_sdk.trace
+ def __init__(self):
+ """
+ Force Stumpy compilation by making a dummy call to the library.
+
+ Note: compilation triggered here is very specific to the exact values of parameters ignore_trivial, normalize etc. A
+ future call with different values for one or more parameter will still trigger a recompilation.
+ """
+ data = np.arange(10.0)
+ mp = stumpy.stump(data, m=3, ignore_trivial=True, normalize=False)
+ stream = stumpy.stumpi(
+ data,
+ m=3,
+ mp=mp,
+ normalize=False,
+ egress=False,
+ )
+ stream.update(6.0)
+
  @sentry_sdk.trace
  def _batch_detect(
  self, timeseries: List[TimeSeriesPoint], config: AnomalyDetectionConfig
@@ -181,6 +202,7 @@ def _update_anomalies(self, ts_external: List[TimeSeriesPoint], anomalies: TimeS
  anomaly_type=anomalies.flags[i],
  )
 
+ @sentry_sdk.trace
  def detect_anomalies(self, request: DetectAnomaliesRequest) -> DetectAnomaliesResponse:
  """
  Main entry point for anomaly detection.
@@ -190,21 +212,22 @@ def detect_anomalies(self, request: DetectAnomaliesRequest) -> DetectAnomaliesRe
  Anomaly detection request that has either a complete time series or an alert reference.
  """
  if isinstance(request.context, AlertInSeer):
- transaction_name = "Stream AD for alert"
+ mode = "streaming.alert"
+ elif isinstance(request.context, TimeSeriesWithHistory):
+ mode = "streaming.ts_with_history"
+ else:
+ mode = "batch.ts_full"
+
+ sentry_sdk.set_tag("ad_mode", mode)
+
+ if isinstance(request.context, AlertInSeer):
+ ts, anomalies = self._online_detect(request.context, request.config)
  elif isinstance(request.context, TimeSeriesWithHistory):
- transaction_name = "Stream AD for timeseries with history"
+ ts, anomalies = self._combo_detect(request.context, request.config)
  else:
- transaction_name = "Batch AD for timeseries"
-
- with sentry_sdk.start_transaction(op="task", name=transaction_name):
- if isinstance(request.context, AlertInSeer):
- ts, anomalies = self._online_detect(request.context, request.config)
- elif isinstance(request.context, TimeSeriesWithHistory):
- ts, anomalies = self._combo_detect(request.context, request.config)
- else:
- ts, anomalies = self._batch_detect(request.context, request.config)
- self._update_anomalies(ts, anomalies)
- return DetectAnomaliesResponse(timeseries=ts)
+ ts, anomalies = self._batch_detect(request.context, request.config)
+ self._update_anomalies(ts, anomalies)
+ return DetectAnomaliesResponse(timeseries=ts)
 
  @inject
  def store_data(

src/seer/bootup.py

@@ -36,10 +36,11 @@ def bootup(
  *, start_model_loading: bool, integrations: list[Integration], config: AppConfig = injected
 ):
  initialize_sentry_sdk(integrations)
- initialize_logs(["seer.", "celery_app."])
- config.do_validation()
- initialize_database()
- initialize_models(start_model_loading)
+ with sentry_sdk.metrics.timing(key="seer_bootup_time"):
+ initialize_logs(["seer.", "celery_app."])
+ config.do_validation()
+ initialize_database()
+ initialize_models(start_model_loading)
 
 
 @inject