Add back Bigeye checks for Thunderbird apps

mozilla · Nov 13, 2024 · 2a6ae85 · 2a6ae85
1 parent 67eca10
commit 2a6ae85
Show file tree

Hide file tree

Showing 10 changed files with 961 additions and 1 deletion.
diff --git a/bigquery_etl/cli/monitoring.py b/bigquery_etl/cli/monitoring.py
@@ -368,7 +368,7 @@ def _update_bigconfig(
     for collection in bigconfig.tag_deployments:
         for deployment in collection.deployments:
             for metric in deployment.metrics:
-                if metric.metric_type.predefined_metric in default_metrics:
+                if metric.metric_type and metric.metric_type.predefined_metric in default_metrics:
                     default_metrics.remove(metric.metric_type.predefined_metric)
 
         if metadata.monitoring.collection and collection.collection is None:

diff --git a/bqetl_project.yaml b/bqetl_project.yaml
@@ -296,6 +296,10 @@ generate:
     - sql/moz-fx-data-shared-prod/org_mozilla_ios_focus_derived/baseline_clients_last_seen_v1/bigconfig.yml
     - sql/moz-fx-data-shared-prod/org_mozilla_social_nightly_derived/baseline_clients_last_seen_v1/checks.sql
     - sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/checks.sql
+    - sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/metadata.yaml
+    - sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/metadata.yaml
+    - sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/metadata.yaml
+    - sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/metadata.yaml
     skip_apps:
     - mlhackweek_search
     - moso_mastodon_android

diff --git a/...red-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/bigconfig.yml b/...red-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/bigconfig.yml
@@ -0,0 +1,205 @@
+type: BIGCONFIG_FILE
+row_creation_times:
+  column_selectors:
+  - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.submission_date
+saved_metric_definitions:
+  metrics:
+  - saved_metric_id: PERCENT_NULL_net_thunderbird_android_beta_derived
+    metric_type:
+      type: PREDEFINED
+      predefined_metric: PERCENT_NULL
+    metric_name: Null (%)
+    group_by:
+    - submission_date
+    threshold:
+      type: CONSTANT
+      upper_bound: 0.0
+      lower_bound: 0.0
+    lookback:
+      lookback_window:
+        interval_type: DAYS
+        interval_value: -1
+      lookback_type: METRIC_TIME
+      bucket_size: DAY
+    metric_schedule:
+      named_schedule:
+        name: Default Schedule - 13:00 UTC
+    rct_overrides:
+    - submission_date
+  - saved_metric_id: COUNT_VALUE_IN_LIST_net_thunderbird_android_beta_derived
+    metric_type:
+      type: PREDEFINED
+      predefined_metric: COUNT_VALUE_IN_LIST
+    metric_name: Value in list (#)
+    group_by:
+    - submission_date
+    threshold:
+      type: AUTO
+      sensitivity: MEDIUM
+      upper_bound_only: false
+      lower_bound_only: false
+    parameters:
+    - key: list
+      string_value: nightly, aurora, release, Other, beta, esr
+      column_name: ''
+      number_value: 0.0
+    lookback:
+      lookback_window:
+        interval_type: DAYS
+        interval_value: -1
+      lookback_type: METRIC_TIME
+      bucket_size: DAY
+    metric_schedule:
+      named_schedule:
+        name: Default Schedule - 13:00 UTC
+    rct_overrides:
+    - submission_date
+  - saved_metric_id: COUNTRY_REGEX_CHECK_net_thunderbird_android_beta_derived
+    metric_type:
+      type: TEMPLATE
+      template_id: 945
+      aggregation_type: COUNT
+      template_name: country_regex_check
+    metric_name: COUNT of country_regex_check
+    group_by:
+    - submission_date
+    threshold:
+      type: AUTO
+      sensitivity: MEDIUM
+      upper_bound_only: false
+      lower_bound_only: false
+    parameters:
+    - key: column
+      string_value: country
+    - key: column_name
+      string_value: country
+    lookback:
+      lookback_window:
+        interval_type: DAYS
+        interval_value: -1
+      lookback_type: METRIC_TIME
+      bucket_size: DAY
+    metric_schedule:
+      named_schedule:
+        name: Default Schedule - 13:00 UTC
+    rct_overrides:
+    - submission_date
+  - saved_metric_id: FRESHNESS_net_thunderbird_android_beta_derived
+    metric_type:
+      type: PREDEFINED
+      predefined_metric: FRESHNESS
+    metric_name: FRESHNESS [warn]
+    threshold:
+      type: AUTO
+      sensitivity: MEDIUM
+      upper_bound_only: true
+      lower_bound_only: false
+    lookback:
+      lookback_window:
+        interval_type: DAYS
+        interval_value: -1
+      lookback_type: METRIC_TIME
+      bucket_size: HOUR
+    metric_schedule:
+      named_schedule:
+        name: Default Schedule - 13:00 UTC
+    rct_overrides:
+    - submission_date
+  - saved_metric_id: VOLUME_net_thunderbird_android_beta_derived
+    metric_type:
+      type: PREDEFINED
+      predefined_metric: VOLUME
+    metric_name: VOLUME [fail]
+    threshold:
+      type: AUTO
+      sensitivity: MEDIUM
+      upper_bound_only: false
+      lower_bound_only: false
+    lookback:
+      lookback_window:
+        interval_type: DAYS
+        interval_value: -1
+      lookback_type: METRIC_TIME
+      bucket_size: HOUR
+    metric_schedule:
+      named_schedule:
+        name: Default Schedule - 13:00 UTC
+    rct_overrides:
+    - submission_date
+  - saved_metric_id: COUNT_DUPLICATES_net_thunderbird_android_beta_derived
+    metric_type:
+      type: PREDEFINED
+      predefined_metric: COUNT_DUPLICATES
+    metric_name: Duplicates (#)
+    group_by:
+    - submission_date
+    threshold:
+      type: AUTO
+      sensitivity: MEDIUM
+      upper_bound_only: false
+      lower_bound_only: false
+    lookback:
+      lookback_window:
+        interval_type: DAYS
+        interval_value: -1
+      lookback_type: METRIC_TIME
+      bucket_size: DAY
+    metric_schedule:
+      named_schedule:
+        name: Default Schedule - 13:00 UTC
+    rct_overrides:
+    - submission_date
+  - saved_metric_id: STRING_LENGTH_MAX_net_thunderbird_android_beta_derived
+    metric_type:
+      type: PREDEFINED
+      predefined_metric: STRING_LENGTH_MAX
+    metric_name: Maximum string length
+    threshold:
+      type: CONSTANT
+      upper_bound: 36.0
+    lookback:
+      lookback_window:
+        interval_type: DAYS
+        interval_value: -1
+      lookback_type: METRIC_TIME
+      bucket_size: DAY
+    metric_schedule:
+      named_schedule:
+        name: Default Schedule - 13:00 UTC
+    rct_overrides:
+    - submission_date
+tag_deployments:
+- collection:
+    name: Thunderbird
+    description: Checks related to thunderbird datasets
+  deployments:
+  - column_selectors:
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.submission_date
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.sample_id
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.first_seen_date
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_seen_session_start_bits
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_seen_session_end_bits
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_seen_bits
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_created_profile_bits
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_active_bits
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.client_id
+    metrics:
+    - saved_metric_id: PERCENT_NULL_net_thunderbird_android_beta_derived
+  - column_selectors:
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.normalized_channel
+    metrics:
+    - saved_metric_id: COUNT_VALUE_IN_LIST_net_thunderbird_android_beta_derived
+  - column_selectors:
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.*
+    metrics:
+    - saved_metric_id: COUNTRY_REGEX_CHECK_net_thunderbird_android_beta_derived
+  - column_selectors:
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.*
+    metrics:
+    - saved_metric_id: FRESHNESS_net_thunderbird_android_beta_derived
+    - saved_metric_id: VOLUME_net_thunderbird_android_beta_derived
+  - column_selectors:
+    - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.client_id
+    metrics:
+    - saved_metric_id: COUNT_DUPLICATES_net_thunderbird_android_beta_derived
+    - saved_metric_id: STRING_LENGTH_MAX_net_thunderbird_android_beta_derived
diff --git a/...red-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/metadata.yaml b/...red-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/metadata.yaml
@@ -0,0 +1,51 @@
+friendly_name: Baseline Clients Last Seen
+description: |-
+  Captures activity history of each client in 28-day
+  windows for each submission date based on baseline pings.
+
+  Clustering fields: `normalized_channel`, `sample_id`
+
+  See also: `baseline_clients_daily`
+owners:
+- ascholtz@mozilla.com
+labels:
+  incremental: true
+  schedule: daily
+  dag: bqetl_glean_usage
+  owner1: ascholtz
+scheduling:
+  dag_name: bqetl_glean_usage
+  task_group: thunderbird_android
+  depends_on_past: true
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: true
+    expiration_days: null
+  range_partitioning: null
+  clustering:
+    fields:
+    - normalized_channel
+    - sample_id
+schema:
+  derived_from:
+  - table:
+    - moz-fx-data-shared-prod
+    - net_thunderbird_android_beta_derived
+    - baseline_clients_daily_v1
+    exclude: null
+workgroup_access:
+- role: roles/bigquery.dataViewer
+  members:
+  - workgroup:mozilla-confidential
+monitoring:
+  enabled: true
+references:
+  checks.sql:
+  - ..
+  - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_daily_v1
+  - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_last_seen_v1
+  query.sql:
+  - net_thunderbird_android_beta_derived.baseline_clients_daily_v1
+  - net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1