diff --git a/bigquery_etl/cli/monitoring.py b/bigquery_etl/cli/monitoring.py index 2f9d24c981d..a02451da010 100644 --- a/bigquery_etl/cli/monitoring.py +++ b/bigquery_etl/cli/monitoring.py @@ -368,7 +368,7 @@ def _update_bigconfig( for collection in bigconfig.tag_deployments: for deployment in collection.deployments: for metric in deployment.metrics: - if metric.metric_type.predefined_metric in default_metrics: + if metric.metric_type and metric.metric_type.predefined_metric in default_metrics: default_metrics.remove(metric.metric_type.predefined_metric) if metadata.monitoring.collection and collection.collection is None: diff --git a/bqetl_project.yaml b/bqetl_project.yaml index ecd9b221ae5..3e303ce8f69 100644 --- a/bqetl_project.yaml +++ b/bqetl_project.yaml @@ -296,6 +296,10 @@ generate: - sql/moz-fx-data-shared-prod/org_mozilla_ios_focus_derived/baseline_clients_last_seen_v1/bigconfig.yml - sql/moz-fx-data-shared-prod/org_mozilla_social_nightly_derived/baseline_clients_last_seen_v1/checks.sql - sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/checks.sql + - sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/metadata.yaml + - sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/metadata.yaml + - sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/metadata.yaml + - sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/metadata.yaml skip_apps: - mlhackweek_search - moso_mastodon_android diff --git a/sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/bigconfig.yml b/sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/bigconfig.yml new file mode 100644 index 00000000000..4823cdb3cbe --- /dev/null +++ b/sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/bigconfig.yml @@ -0,0 +1,205 @@ +type: BIGCONFIG_FILE +row_creation_times: + column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.submission_date +saved_metric_definitions: + metrics: + - saved_metric_id: PERCENT_NULL_net_thunderbird_android_beta_derived + metric_type: + type: PREDEFINED + predefined_metric: PERCENT_NULL + metric_name: Null (%) + group_by: + - submission_date + threshold: + type: CONSTANT + upper_bound: 0.0 + lower_bound: 0.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNT_VALUE_IN_LIST_net_thunderbird_android_beta_derived + metric_type: + type: PREDEFINED + predefined_metric: COUNT_VALUE_IN_LIST + metric_name: Value in list (#) + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + parameters: + - key: list + string_value: nightly, aurora, release, Other, beta, esr + column_name: '' + number_value: 0.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNTRY_REGEX_CHECK_net_thunderbird_android_beta_derived + metric_type: + type: TEMPLATE + template_id: 945 + aggregation_type: COUNT + template_name: country_regex_check + metric_name: COUNT of country_regex_check + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + parameters: + - key: column + string_value: country + - key: column_name + string_value: country + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: FRESHNESS_net_thunderbird_android_beta_derived + metric_type: + type: PREDEFINED + predefined_metric: FRESHNESS + metric_name: FRESHNESS [warn] + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: true + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: HOUR + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: VOLUME_net_thunderbird_android_beta_derived + metric_type: + type: PREDEFINED + predefined_metric: VOLUME + metric_name: VOLUME [fail] + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: HOUR + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNT_DUPLICATES_net_thunderbird_android_beta_derived + metric_type: + type: PREDEFINED + predefined_metric: COUNT_DUPLICATES + metric_name: Duplicates (#) + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: STRING_LENGTH_MAX_net_thunderbird_android_beta_derived + metric_type: + type: PREDEFINED + predefined_metric: STRING_LENGTH_MAX + metric_name: Maximum string length + threshold: + type: CONSTANT + upper_bound: 36.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date +tag_deployments: +- collection: + name: Thunderbird + description: Checks related to thunderbird datasets + deployments: + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.submission_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.sample_id + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.first_seen_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_seen_session_start_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_seen_session_end_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_seen_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_created_profile_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.days_active_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.client_id + metrics: + - saved_metric_id: PERCENT_NULL_net_thunderbird_android_beta_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.normalized_channel + metrics: + - saved_metric_id: COUNT_VALUE_IN_LIST_net_thunderbird_android_beta_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.* + metrics: + - saved_metric_id: COUNTRY_REGEX_CHECK_net_thunderbird_android_beta_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.* + metrics: + - saved_metric_id: FRESHNESS_net_thunderbird_android_beta_derived + - saved_metric_id: VOLUME_net_thunderbird_android_beta_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1.client_id + metrics: + - saved_metric_id: COUNT_DUPLICATES_net_thunderbird_android_beta_derived + - saved_metric_id: STRING_LENGTH_MAX_net_thunderbird_android_beta_derived diff --git a/sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/metadata.yaml new file mode 100644 index 00000000000..c8ba921aff3 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/net_thunderbird_android_beta_derived/baseline_clients_last_seen_v1/metadata.yaml @@ -0,0 +1,51 @@ +friendly_name: Baseline Clients Last Seen +description: |- + Captures activity history of each client in 28-day + windows for each submission date based on baseline pings. + + Clustering fields: `normalized_channel`, `sample_id` + + See also: `baseline_clients_daily` +owners: +- ascholtz@mozilla.com +labels: + incremental: true + schedule: daily + dag: bqetl_glean_usage + owner1: ascholtz +scheduling: + dag_name: bqetl_glean_usage + task_group: thunderbird_android + depends_on_past: true +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: true + expiration_days: null + range_partitioning: null + clustering: + fields: + - normalized_channel + - sample_id +schema: + derived_from: + - table: + - moz-fx-data-shared-prod + - net_thunderbird_android_beta_derived + - baseline_clients_daily_v1 + exclude: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +monitoring: + enabled: true +references: + checks.sql: + - .. + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_daily_v1 + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_last_seen_v1 + query.sql: + - net_thunderbird_android_beta_derived.baseline_clients_daily_v1 + - net_thunderbird_android_beta_derived.baseline_clients_last_seen_v1 diff --git a/sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/bigconfig.yml b/sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/bigconfig.yml new file mode 100644 index 00000000000..506d16bc12c --- /dev/null +++ b/sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/bigconfig.yml @@ -0,0 +1,139 @@ +type: BIGCONFIG_FILE +row_creation_times: + column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.submission_date +saved_metric_definitions: + metrics: + - saved_metric_id: PERCENT_NULL_net_thunderbird_android_daily_derived + metric_type: + type: PREDEFINED + predefined_metric: PERCENT_NULL + metric_name: Null (%) + group_by: + - submission_date + threshold: + type: CONSTANT + upper_bound: 0.0 + lower_bound: 0.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: FRESHNESS_net_thunderbird_android_daily_derived + metric_type: + type: PREDEFINED + predefined_metric: FRESHNESS + metric_name: FRESHNESS [warn] + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: true + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: HOUR + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: VOLUME_net_thunderbird_android_daily_derived + metric_type: + type: PREDEFINED + predefined_metric: VOLUME + metric_name: VOLUME [fail] + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: HOUR + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNT_DUPLICATES_net_thunderbird_android_daily_derived + metric_type: + type: PREDEFINED + predefined_metric: COUNT_DUPLICATES + metric_name: Duplicates (#) + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: STRING_LENGTH_MAX_net_thunderbird_android_daily_derived + metric_type: + type: PREDEFINED + predefined_metric: STRING_LENGTH_MAX + metric_name: Maximum string length + threshold: + type: CONSTANT + upper_bound: 36.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date +tag_deployments: +- collection: + name: Thunderbird + description: Alerts related to all applicable thunderbird datasets + deployments: + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.submission_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.sample_id + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.first_seen_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.days_seen_session_start_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.days_seen_session_end_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.days_seen_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.days_created_profile_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.days_active_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.client_id + metrics: + - saved_metric_id: PERCENT_NULL_net_thunderbird_android_daily_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.* + metrics: + - saved_metric_id: FRESHNESS_net_thunderbird_android_daily_derived + - saved_metric_id: VOLUME_net_thunderbird_android_daily_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1.client_id + metrics: + - saved_metric_id: COUNT_DUPLICATES_net_thunderbird_android_daily_derived + - saved_metric_id: STRING_LENGTH_MAX_net_thunderbird_android_daily_derived diff --git a/sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/metadata.yaml new file mode 100644 index 00000000000..8c386e0f172 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/net_thunderbird_android_daily_derived/baseline_clients_last_seen_v1/metadata.yaml @@ -0,0 +1,49 @@ +friendly_name: Baseline Clients Last Seen +description: |- + Captures activity history of each client in 28-day + windows for each submission date based on baseline pings. + + Clustering fields: `normalized_channel`, `sample_id` + + See also: `baseline_clients_daily` +owners: +- ascholtz@mozilla.com +labels: + incremental: true + schedule: daily + dag: bqetl_glean_usage + owner1: ascholtz +scheduling: + dag_name: bqetl_glean_usage + task_group: thunderbird_android + depends_on_past: true +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: true + expiration_days: null + range_partitioning: null + clustering: + fields: + - normalized_channel + - sample_id +schema: + derived_from: + - table: + - moz-fx-data-shared-prod + - net_thunderbird_android_daily_derived + - baseline_clients_daily_v1 + exclude: null +monitoring: + enabled: true +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + checks.sql: + - .. + query.sql: + - net_thunderbird_android_daily_derived.baseline_clients_daily_v1 + - net_thunderbird_android_daily_derived.baseline_clients_last_seen_v1 diff --git a/sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/bigconfig.yml b/sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/bigconfig.yml new file mode 100644 index 00000000000..c4c3d70496d --- /dev/null +++ b/sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/bigconfig.yml @@ -0,0 +1,205 @@ +type: BIGCONFIG_FILE +row_creation_times: + column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.submission_date +saved_metric_definitions: + metrics: + - saved_metric_id: PERCENT_NULL_net_thunderbird_android_derived + metric_type: + type: PREDEFINED + predefined_metric: PERCENT_NULL + metric_name: Null (%) + group_by: + - submission_date + threshold: + type: CONSTANT + upper_bound: 0.0 + lower_bound: 0.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNT_VALUE_IN_LIST_net_thunderbird_android_derived + metric_type: + type: PREDEFINED + predefined_metric: COUNT_VALUE_IN_LIST + metric_name: Value in list (#) + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + parameters: + - key: list + string_value: nightly, aurora, release, Other, beta, esr + column_name: '' + number_value: 0.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNTRY_REGEX_CHECK_net_thunderbird_android_derived + metric_type: + type: TEMPLATE + template_id: 945 + aggregation_type: COUNT + template_name: country_regex_check + metric_name: COUNT of country_regex_check + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + parameters: + - key: column + string_value: country + - key: column_name + string_value: country + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: FRESHNESS_net_thunderbird_android_derived + metric_type: + type: PREDEFINED + predefined_metric: FRESHNESS + metric_name: FRESHNESS [warn + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: true + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: HOUR + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: VOLUME_net_thunderbird_android_derived + metric_type: + type: PREDEFINED + predefined_metric: VOLUME + metric_name: VOLUME [Fail] + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: HOUR + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNT_DUPLICATES_net_thunderbird_android_derived + metric_type: + type: PREDEFINED + predefined_metric: COUNT_DUPLICATES + metric_name: Duplicates (#) + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: STRING_LENGTH_MAX_net_thunderbird_android_derived + metric_type: + type: PREDEFINED + predefined_metric: STRING_LENGTH_MAX + metric_name: Maximum string length + threshold: + type: CONSTANT + upper_bound: 36.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date +tag_deployments: +- collection: + name: Thunderbird + description: Checks related to thunderbird datasets + deployments: + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.submission_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.sample_id + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.first_seen_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.days_seen_session_start_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.days_seen_session_end_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.days_seen_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.days_created_profile_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.days_active_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.client_id + metrics: + - saved_metric_id: PERCENT_NULL_net_thunderbird_android_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.normalized_channel + metrics: + - saved_metric_id: COUNT_VALUE_IN_LIST_net_thunderbird_android_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.* + metrics: + - saved_metric_id: COUNTRY_REGEX_CHECK_net_thunderbird_android_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.* + metrics: + - saved_metric_id: FRESHNESS_net_thunderbird_android_derived + - saved_metric_id: VOLUME_net_thunderbird_android_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.net_thunderbird_android_derived.baseline_clients_last_seen_v1.client_id + metrics: + - saved_metric_id: COUNT_DUPLICATES_net_thunderbird_android_derived + - saved_metric_id: STRING_LENGTH_MAX_net_thunderbird_android_derived diff --git a/sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/metadata.yaml new file mode 100644 index 00000000000..23023fab005 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/net_thunderbird_android_derived/baseline_clients_last_seen_v1/metadata.yaml @@ -0,0 +1,51 @@ +friendly_name: Baseline Clients Last Seen +description: |- + Captures activity history of each client in 28-day + windows for each submission date based on baseline pings. + + Clustering fields: `normalized_channel`, `sample_id` + + See also: `baseline_clients_daily` +owners: +- ascholtz@mozilla.com +labels: + incremental: true + schedule: daily + dag: bqetl_glean_usage + owner1: ascholtz +scheduling: + dag_name: bqetl_glean_usage + task_group: thunderbird_android + depends_on_past: true +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: true + expiration_days: null + range_partitioning: null + clustering: + fields: + - normalized_channel + - sample_id +schema: + derived_from: + - table: + - moz-fx-data-shared-prod + - net_thunderbird_android_derived + - baseline_clients_daily_v1 + exclude: null +monitoring: + enabled: true +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + checks.sql: + - .. + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_daily_v1 + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_last_seen_v1 + query.sql: + - net_thunderbird_android_derived.baseline_clients_daily_v1 + - net_thunderbird_android_derived.baseline_clients_last_seen_v1 diff --git a/sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/bigconfig.yml b/sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/bigconfig.yml new file mode 100644 index 00000000000..26e9ec62232 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/bigconfig.yml @@ -0,0 +1,205 @@ +type: BIGCONFIG_FILE +row_creation_times: + column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.submission_date +saved_metric_definitions: + metrics: + - saved_metric_id: PERCENT_NULL_thunderbird_desktop_derived + metric_type: + type: PREDEFINED + predefined_metric: PERCENT_NULL + metric_name: Null (%) + group_by: + - submission_date + threshold: + type: CONSTANT + upper_bound: 0.0 + lower_bound: 0.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNT_VALUE_IN_LIST_thunderbird_desktop_derived + metric_type: + type: PREDEFINED + predefined_metric: COUNT_VALUE_IN_LIST + metric_name: Value in list (#) + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + parameters: + - key: list + string_value: nightly, aurora, release, Other, beta, esr + column_name: '' + number_value: 0.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNTRY_REGEX_CHECK_thunderbird_desktop_derived + metric_type: + type: TEMPLATE + template_id: 945 + aggregation_type: COUNT + template_name: country_regex_check + metric_name: COUNT of country_regex_check + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + parameters: + - key: column + string_value: country + - key: column_name + string_value: country + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: FRESHNESS_thunderbird_desktop_derived + metric_type: + type: PREDEFINED + predefined_metric: FRESHNESS + metric_name: FRESHNESS [warn] + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: true + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: HOUR + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: VOLUME_thunderbird_desktop_derived + metric_type: + type: PREDEFINED + predefined_metric: VOLUME + metric_name: VOLUME [fail] + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: HOUR + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: COUNT_DUPLICATES_thunderbird_desktop_derived + metric_type: + type: PREDEFINED + predefined_metric: COUNT_DUPLICATES + metric_name: Duplicates (#) + group_by: + - submission_date + threshold: + type: AUTO + sensitivity: MEDIUM + upper_bound_only: false + lower_bound_only: false + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date + - saved_metric_id: STRING_LENGTH_MAX_thunderbird_desktop_derived + metric_type: + type: PREDEFINED + predefined_metric: STRING_LENGTH_MAX + metric_name: Maximum string length + threshold: + type: CONSTANT + upper_bound: 36.0 + lookback: + lookback_window: + interval_type: DAYS + interval_value: -1 + lookback_type: METRIC_TIME + bucket_size: DAY + metric_schedule: + named_schedule: + name: Default Schedule - 13:00 UTC + rct_overrides: + - submission_date +tag_deployments: +- collection: + name: Thunderbird + description: Checks related to thunderbird datasets + deployments: + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.submission_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.sample_id + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.first_seen_date + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.days_seen_session_start_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.days_seen_session_end_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.days_seen_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.days_created_profile_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.days_active_bits + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.client_id + metrics: + - saved_metric_id: PERCENT_NULL_thunderbird_desktop_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.normalized_channel + metrics: + - saved_metric_id: COUNT_VALUE_IN_LIST_thunderbird_desktop_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.* + metrics: + - saved_metric_id: COUNTRY_REGEX_CHECK_thunderbird_desktop_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.* + metrics: + - saved_metric_id: FRESHNESS_thunderbird_desktop_derived + - saved_metric_id: VOLUME_thunderbird_desktop_derived + - column_selectors: + - name: moz-fx-data-shared-prod.moz-fx-data-shared-prod.thunderbird_desktop_derived.baseline_clients_last_seen_v1.client_id + metrics: + - saved_metric_id: COUNT_DUPLICATES_thunderbird_desktop_derived + - saved_metric_id: STRING_LENGTH_MAX_thunderbird_desktop_derived diff --git a/sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/metadata.yaml b/sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/metadata.yaml new file mode 100644 index 00000000000..aaf67ab0ab7 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/thunderbird_desktop_derived/baseline_clients_last_seen_v1/metadata.yaml @@ -0,0 +1,51 @@ +friendly_name: Baseline Clients Last Seen +description: |- + Captures activity history of each client in 28-day + windows for each submission date based on baseline pings. + + Clustering fields: `normalized_channel`, `sample_id` + + See also: `baseline_clients_daily` +owners: +- ascholtz@mozilla.com +labels: + incremental: true + schedule: daily + dag: bqetl_glean_usage + owner1: ascholtz +scheduling: + dag_name: bqetl_glean_usage + task_group: thunderbird_desktop + depends_on_past: true +bigquery: + time_partitioning: + type: day + field: submission_date + require_partition_filter: true + expiration_days: null + range_partitioning: null + clustering: + fields: + - normalized_channel + - sample_id +schema: + derived_from: + - table: + - moz-fx-data-shared-prod + - thunderbird_desktop_derived + - baseline_clients_daily_v1 + exclude: null +monitoring: + enabled: true +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + checks.sql: + - .. + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_daily_v1 + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_last_seen_v1 + query.sql: + - thunderbird_desktop_derived.baseline_clients_daily_v1 + - thunderbird_desktop_derived.baseline_clients_last_seen_v1