From e97472e1443fcd155f3cb6a6a4d9db2d30b6835d Mon Sep 17 00:00:00 2001 From: Anna Scholtz Date: Wed, 13 Nov 2024 15:13:10 -0800 Subject: [PATCH 1/3] Add back Bigeye checks for Thunderbird apps --- bigquery_etl/cli/monitoring.py | 2 +- bqetl_project.yaml | 4 + .../bigconfig.yml | 205 ++++++++++++++++++ .../metadata.yaml | 51 +++++ .../bigconfig.yml | 139 ++++++++++++ .../metadata.yaml | 49 +++++ .../bigconfig.yml | 205 ++++++++++++++++++ .../metadata.yaml | 51 +++++ .../bigconfig.yml | 205 ++++++++++++++++++ .../metadata.yaml | 51 +++++ 10 files changed, 961 insertions(+), 1 deletion(-) create mode 100644 sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/bigconfig.yml create mode 100644 sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/bigconfig.yml create mode 100644 sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/bigconfig.yml create mode 100644 sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/metadata.yaml create mode 100644 sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/bigconfig.yml create mode 100644 sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/metadata.yaml diff --git a/bigquery_etl/cli/monitoring.py b/bigquery_etl/cli/monitoring.py index 8eb57b62cab..9eb17761df7 100644 --- a/bigquery_etl/cli/monitoring.py +++ b/bigquery_etl/cli/monitoring.py @@ -376,7 +376,7 @@ def _update_bigconfig( raise Exception(err_message) - if metric.metric_type.predefined_metric in default_metrics: + if metric.metric_type and metric.metric_type.predefined_metric in default_metrics: default_metrics.remove(metric.metric_type.predefined_metric) if metadata.monitoring.collection and collection.collection is None: diff --git a/bqetl_project.yaml b/bqetl_project.yaml index ecd9b221ae5..3e303ce8f69 100644 --- a/bqetl_project.yaml +++ b/bqetl_project.yaml @@ -296,6 +296,10 @@ generate: - sql/moz-fx-data-shared-prod/org_mozilla_ios_focus_derived/baseline_clients_last_seen_v1/bigconfig.yml - sql/moz-fx-data-shared-prod/org_mozilla_social_nightly_derived/baseline_clients_last_seen_v1/checks.sql - sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/checks.sql + - sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/metadata.yaml + - sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/metadata.yaml + - sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/metadata.yaml + - sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/metadata.yaml skip_apps: - mlhackweek_search - moso_mastodon_android diff --git a/sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/bigconfig.yml b/sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/bigconfig.yml new file mode 100644 index 00000000000..4823cdb3cbe --- /dev/null +++ b/sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/bigconfig.yml @@ -0,0 +1,205 @@ +type: BIGCONFIG_FILE +row_creation_times: + column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.submission_date +saved_metric_definitions: + metrics: + - saved_metric_id: PERCENT_NULL_net_thunderbird_android_beta_derived + metric_type: + type: PREDEFINED + predefined_metric: PERCENT_NULL + metric_name: Null (%) + group_by: + - submission_date + threshold: + type: CONSTANT + upper_bound: 0.0 + lower_bound: 0.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNT_VALUE_IN_LIST_net_thunderbird_android_beta_derived + metric_type: + type: PREDEFINED + predefined_metric: COUNT_VALUE_IN_LIST + metric_name: Value in list (#) + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + parameters: + - key: list + string_value: nightly, aurora, release, Other, beta, esr + column_name: '' + number_value: 0.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNTRY_REGEX_CHECK_net_thunderbird_android_beta_derived + metric_type: + type: TEMPLATE + template_id: 945 + aggregation_type: COUNT + template_name: country_regex_check + metric_name: COUNT of country_regex_check + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + parameters: + - key: column + string_value: country + - key: column_name + string_value: country + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: FRESHNESS_net_thunderbird_android_beta_derived + metric_type: + type: PREDEFINED + predefined_metric: FRESHNESS + metric_name: FRESHNESS [warn] + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: true + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: HOUR + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: VOLUME_net_thunderbird_android_beta_derived + metric_type: + type: PREDEFINED + predefined_metric: VOLUME + metric_name: VOLUME [fail] + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: HOUR + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNT_DUPLICATES_net_thunderbird_android_beta_derived + metric_type: + type: PREDEFINED + predefined_metric: COUNT_DUPLICATES + metric_name: Duplicates (#) + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: STRING_LENGTH_MAX_net_thunderbird_android_beta_derived + metric_type: + type: PREDEFINED + predefined_metric: STRING_LENGTH_MAX + metric_name: Maximum string length + threshold: + type: CONSTANT + upper_bound: 36.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date +tag_deployments: +- collection: + name: Thunderbird + description: Checks related to thunderbird datasets + deployments: + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.submission_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.sample_id + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.first_seen_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_seen_session_start_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_seen_session_end_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_seen_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_created_profile_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_active_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.client_id + metrics: + - saved_metric_id: PERCENT_NULL_net_thunderbird_android_beta_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.normalized_channel + metrics: + - saved_metric_id: COUNT_VALUE_IN_LIST_net_thunderbird_android_beta_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.* + metrics: + - saved_metric_id: COUNTRY_REGEX_CHECK_net_thunderbird_android_beta_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.* + metrics: + - saved_metric_id: FRESHNESS_net_thunderbird_android_beta_derived + - saved_metric_id: VOLUME_net_thunderbird_android_beta_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.client_id + metrics: + - saved_metric_id: COUNT_DUPLICATES_net_thunderbird_android_beta_derived + - saved_metric_id: STRING_LENGTH_MAX_net_thunderbird_android_beta_derived diff --git a/sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/metadata.yaml new file mode 100644 index 00000000000..c8ba921aff3 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/metadata.yaml @@ -0,0 +1,51 @@ +friendly_name: Baseline Clients Last Seen +description: |- + Captures activity history of each client in 28-day + windows for each submission date based on baseline pings. + + Clustering fields: `normalized_channel`, `sample_id` + + See also: `baseline_clients_daily` +owners: +- ascholtz@mozilla.com +labels: + incremental: true + schedule: daily + dag: bqetl_glean_usage + owner1: ascholtz +scheduling: + dag_name: bqetl_glean_usage + task_group: thunderbird_android + depends_on_past: true +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: true + expiration_days: null + range_partitioning: null + clustering: + fields: + - normalized_channel + - sample_id +schema: + derived_from: + - table: + - moz-fx-data-shared-prod + - net_thunderbird_android_beta_derived + - baseline_clients_daily_v1 + exclude: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +monitoring: + enabled: true +references: + checks.sql: + - .. + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_daily_v1 + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_last_seen_v1 + query.sql: + - net_thunderbird_android_beta_derived.baseline_clients_daily_v1 + - net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1 diff --git a/sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/bigconfig.yml b/sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/bigconfig.yml new file mode 100644 index 00000000000..506d16bc12c --- /dev/null +++ b/sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/bigconfig.yml @@ -0,0 +1,139 @@ +type: BIGCONFIG_FILE +row_creation_times: + column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.submission_date +saved_metric_definitions: + metrics: + - saved_metric_id: PERCENT_NULL_net_thunderbird_android_daily_derived + metric_type: + type: PREDEFINED + predefined_metric: PERCENT_NULL + metric_name: Null (%) + group_by: + - submission_date + threshold: + type: CONSTANT + upper_bound: 0.0 + lower_bound: 0.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: FRESHNESS_net_thunderbird_android_daily_derived + metric_type: + type: PREDEFINED + predefined_metric: FRESHNESS + metric_name: FRESHNESS [warn] + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: true + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: HOUR + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: VOLUME_net_thunderbird_android_daily_derived + metric_type: + type: PREDEFINED + predefined_metric: VOLUME + metric_name: VOLUME [fail] + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: HOUR + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNT_DUPLICATES_net_thunderbird_android_daily_derived + metric_type: + type: PREDEFINED + predefined_metric: COUNT_DUPLICATES + metric_name: Duplicates (#) + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: STRING_LENGTH_MAX_net_thunderbird_android_daily_derived + metric_type: + type: PREDEFINED + predefined_metric: STRING_LENGTH_MAX + metric_name: Maximum string length + threshold: + type: CONSTANT + upper_bound: 36.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date +tag_deployments: +- collection: + name: Thunderbird + description: Alerts related to all applicable thunderbird datasets + deployments: + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.submission_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.sample_id + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.first_seen_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.days_seen_session_start_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.days_seen_session_end_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.days_seen_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.days_created_profile_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.days_active_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.client_id + metrics: + - saved_metric_id: PERCENT_NULL_net_thunderbird_android_daily_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.* + metrics: + - saved_metric_id: FRESHNESS_net_thunderbird_android_daily_derived + - saved_metric_id: VOLUME_net_thunderbird_android_daily_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.client_id + metrics: + - saved_metric_id: COUNT_DUPLICATES_net_thunderbird_android_daily_derived + - saved_metric_id: STRING_LENGTH_MAX_net_thunderbird_android_daily_derived diff --git a/sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/metadata.yaml new file mode 100644 index 00000000000..8c386e0f172 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/metadata.yaml @@ -0,0 +1,49 @@ +friendly_name: Baseline Clients Last Seen +description: |- + Captures activity history of each client in 28-day + windows for each submission date based on baseline pings. + + Clustering fields: `normalized_channel`, `sample_id` + + See also: `baseline_clients_daily` +owners: +- ascholtz@mozilla.com +labels: + incremental: true + schedule: daily + dag: bqetl_glean_usage + owner1: ascholtz +scheduling: + dag_name: bqetl_glean_usage + task_group: thunderbird_android + depends_on_past: true +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: true + expiration_days: null + range_partitioning: null + clustering: + fields: + - normalized_channel + - sample_id +schema: + derived_from: + - table: + - moz-fx-data-shared-prod + - net_thunderbird_android_daily_derived + - baseline_clients_daily_v1 + exclude: null +monitoring: + enabled: true +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + checks.sql: + - .. + query.sql: + - net_thunderbird_android_daily_derived.baseline_clients_daily_v1 + - net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1 diff --git a/sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/bigconfig.yml b/sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/bigconfig.yml new file mode 100644 index 00000000000..c4c3d70496d --- /dev/null +++ b/sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/bigconfig.yml @@ -0,0 +1,205 @@ +type: BIGCONFIG_FILE +row_creation_times: + column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.submission_date +saved_metric_definitions: + metrics: + - saved_metric_id: PERCENT_NULL_net_thunderbird_android_derived + metric_type: + type: PREDEFINED + predefined_metric: PERCENT_NULL + metric_name: Null (%) + group_by: + - submission_date + threshold: + type: CONSTANT + upper_bound: 0.0 + lower_bound: 0.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNT_VALUE_IN_LIST_net_thunderbird_android_derived + metric_type: + type: PREDEFINED + predefined_metric: COUNT_VALUE_IN_LIST + metric_name: Value in list (#) + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + parameters: + - key: list + string_value: nightly, aurora, release, Other, beta, esr + column_name: '' + number_value: 0.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNTRY_REGEX_CHECK_net_thunderbird_android_derived + metric_type: + type: TEMPLATE + template_id: 945 + aggregation_type: COUNT + template_name: country_regex_check + metric_name: COUNT of country_regex_check + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + parameters: + - key: column + string_value: country + - key: column_name + string_value: country + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: FRESHNESS_net_thunderbird_android_derived + metric_type: + type: PREDEFINED + predefined_metric: FRESHNESS + metric_name: FRESHNESS [warn + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: true + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: HOUR + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: VOLUME_net_thunderbird_android_derived + metric_type: + type: PREDEFINED + predefined_metric: VOLUME + metric_name: VOLUME [Fail] + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: HOUR + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNT_DUPLICATES_net_thunderbird_android_derived + metric_type: + type: PREDEFINED + predefined_metric: COUNT_DUPLICATES + metric_name: Duplicates (#) + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: STRING_LENGTH_MAX_net_thunderbird_android_derived + metric_type: + type: PREDEFINED + predefined_metric: STRING_LENGTH_MAX + metric_name: Maximum string length + threshold: + type: CONSTANT + upper_bound: 36.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date +tag_deployments: +- collection: + name: Thunderbird + description: Checks related to thunderbird datasets + deployments: + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.submission_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.sample_id + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.first_seen_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.days_seen_session_start_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.days_seen_session_end_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.days_seen_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.days_created_profile_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.days_active_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.client_id + metrics: + - saved_metric_id: PERCENT_NULL_net_thunderbird_android_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.normalized_channel + metrics: + - saved_metric_id: COUNT_VALUE_IN_LIST_net_thunderbird_android_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.* + metrics: + - saved_metric_id: COUNTRY_REGEX_CHECK_net_thunderbird_android_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.* + metrics: + - saved_metric_id: FRESHNESS_net_thunderbird_android_derived + - saved_metric_id: VOLUME_net_thunderbird_android_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.client_id + metrics: + - saved_metric_id: COUNT_DUPLICATES_net_thunderbird_android_derived + - saved_metric_id: STRING_LENGTH_MAX_net_thunderbird_android_derived diff --git a/sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/metadata.yaml new file mode 100644 index 00000000000..23023fab005 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/metadata.yaml @@ -0,0 +1,51 @@ +friendly_name: Baseline Clients Last Seen +description: |- + Captures activity history of each client in 28-day + windows for each submission date based on baseline pings. + + Clustering fields: `normalized_channel`, `sample_id` + + See also: `baseline_clients_daily` +owners: +- ascholtz@mozilla.com +labels: + incremental: true + schedule: daily + dag: bqetl_glean_usage + owner1: ascholtz +scheduling: + dag_name: bqetl_glean_usage + task_group: thunderbird_android + depends_on_past: true +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: true + expiration_days: null + range_partitioning: null + clustering: + fields: + - normalized_channel + - sample_id +schema: + derived_from: + - table: + - moz-fx-data-shared-prod + - net_thunderbird_android_derived + - baseline_clients_daily_v1 + exclude: null +monitoring: + enabled: true +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + checks.sql: + - .. + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_daily_v1 + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_last_seen_v1 + query.sql: + - net_thunderbird_android_derived.baseline_clients_daily_v1 + - net_thunderbird_android_derived.baseline_clients_last_seen_v1 diff --git a/sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/bigconfig.yml b/sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/bigconfig.yml new file mode 100644 index 00000000000..26e9ec62232 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/bigconfig.yml @@ -0,0 +1,205 @@ +type: BIGCONFIG_FILE +row_creation_times: + column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.submission_date +saved_metric_definitions: + metrics: + - saved_metric_id: PERCENT_NULL_thunderbird_desktop_derived + metric_type: + type: PREDEFINED + predefined_metric: PERCENT_NULL + metric_name: Null (%) + group_by: + - submission_date + threshold: + type: CONSTANT + upper_bound: 0.0 + lower_bound: 0.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNT_VALUE_IN_LIST_thunderbird_desktop_derived + metric_type: + type: PREDEFINED + predefined_metric: COUNT_VALUE_IN_LIST + metric_name: Value in list (#) + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + parameters: + - key: list + string_value: nightly, aurora, release, Other, beta, esr + column_name: '' + number_value: 0.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNTRY_REGEX_CHECK_thunderbird_desktop_derived + metric_type: + type: TEMPLATE + template_id: 945 + aggregation_type: COUNT + template_name: country_regex_check + metric_name: COUNT of country_regex_check + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + parameters: + - key: column + string_value: country + - key: column_name + string_value: country + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: FRESHNESS_thunderbird_desktop_derived + metric_type: + type: PREDEFINED + predefined_metric: FRESHNESS + metric_name: FRESHNESS [warn] + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: true + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: HOUR + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: VOLUME_thunderbird_desktop_derived + metric_type: + type: PREDEFINED + predefined_metric: VOLUME + metric_name: VOLUME [fail] + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: HOUR + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNT_DUPLICATES_thunderbird_desktop_derived + metric_type: + type: PREDEFINED + predefined_metric: COUNT_DUPLICATES + metric_name: Duplicates (#) + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: STRING_LENGTH_MAX_thunderbird_desktop_derived + metric_type: + type: PREDEFINED + predefined_metric: STRING_LENGTH_MAX + metric_name: Maximum string length + threshold: + type: CONSTANT + upper_bound: 36.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date +tag_deployments: +- collection: + name: Thunderbird + description: Checks related to thunderbird datasets + deployments: + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.submission_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.sample_id + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.first_seen_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.days_seen_session_start_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.days_seen_session_end_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.days_seen_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.days_created_profile_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.days_active_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.client_id + metrics: + - saved_metric_id: PERCENT_NULL_thunderbird_desktop_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.normalized_channel + metrics: + - saved_metric_id: COUNT_VALUE_IN_LIST_thunderbird_desktop_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.* + metrics: + - saved_metric_id: COUNTRY_REGEX_CHECK_thunderbird_desktop_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.* + metrics: + - saved_metric_id: FRESHNESS_thunderbird_desktop_derived + - saved_metric_id: VOLUME_thunderbird_desktop_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.client_id + metrics: + - saved_metric_id: COUNT_DUPLICATES_thunderbird_desktop_derived + - saved_metric_id: STRING_LENGTH_MAX_thunderbird_desktop_derived diff --git a/sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/metadata.yaml new file mode 100644 index 00000000000..aaf67ab0ab7 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/metadata.yaml @@ -0,0 +1,51 @@ +friendly_name: Baseline Clients Last Seen +description: |- + Captures activity history of each client in 28-day + windows for each submission date based on baseline pings. + + Clustering fields: `normalized_channel`, `sample_id` + + See also: `baseline_clients_daily` +owners: +- ascholtz@mozilla.com +labels: + incremental: true + schedule: daily + dag: bqetl_glean_usage + owner1: ascholtz +scheduling: + dag_name: bqetl_glean_usage + task_group: thunderbird_desktop + depends_on_past: true +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: true + expiration_days: null + range_partitioning: null + clustering: + fields: + - normalized_channel + - sample_id +schema: + derived_from: + - table: + - moz-fx-data-shared-prod + - thunderbird_desktop_derived + - baseline_clients_daily_v1 + exclude: null +monitoring: + enabled: true +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + checks.sql: + - .. + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_daily_v1 + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_last_seen_v1 + query.sql: + - thunderbird_desktop_derived.baseline_clients_daily_v1 + - thunderbird_desktop_derived.baseline_clients_last_seen_v1 From 13af5b2d3151bc5b830ce3d24261a1d53ff90e64 Mon Sep 17 00:00:00 2001 From: Anna Scholtz Date: Thu, 14 Nov 2024 14:09:59 -0800 Subject: [PATCH 2/3] Reformat --- bigquery_etl/cli/monitoring.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bigquery_etl/cli/monitoring.py b/bigquery_etl/cli/monitoring.py index 9eb17761df7..0531e248a54 100644 --- a/bigquery_etl/cli/monitoring.py +++ b/bigquery_etl/cli/monitoring.py @@ -376,7 +376,10 @@ def _update_bigconfig( raise Exception(err_message) - if metric.metric_type and metric.metric_type.predefined_metric in default_metrics: + if ( + metric.metric_type + and metric.metric_type.predefined_metric in default_metrics + ): default_metrics.remove(metric.metric_type.predefined_metric) if metadata.monitoring.collection and collection.collection is None: From cc0f32b894da7b0927815adf58e89cd88777b583 Mon Sep 17 00:00:00 2001 From: Anna Scholtz Date: Thu, 14 Nov 2024 14:43:35 -0800 Subject: [PATCH 3/3] Disable monitoring for release --- sql/moz-fx-data-shared-prod/telemetry/releases/metadata.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/telemetry/releases/metadata.yaml b/sql/moz-fx-data-shared-prod/telemetry/releases/metadata.yaml index 6f2417a5d1b..14f872f308d 100644 --- a/sql/moz-fx-data-shared-prod/telemetry/releases/metadata.yaml +++ b/sql/moz-fx-data-shared-prod/telemetry/releases/metadata.yaml @@ -7,6 +7,4 @@ description: |- https://wiki.mozilla.org/Release_Management/Product_details owners: - ascholtz@mozilla.com -monitoring: - enabled: true - partition_column: date +