RS_1233 add is_enterprise_policies to search_clients_daily_v8 and search_aggregates

[pissac17]

Checklist for reviewer:

• [ ] Commits should reference a bug or github issue, if relevant (if a bug is referenced, the pull request should include the bug number in the title).
• [ ] If the PR comes from a fork, trigger integration CI tests by running the Push to upstream workflow and provide the <username>:<branch> of the fork as parameter. The parameter will also show up in the logs of the manual-trigger-required-for-fork CI task together with more detailed instructions.
• [ ] If adding a new field to a query, ensure that the schema and dependent downstream schemas have been updated.
• [ ] When adding a new derived dataset, ensure that data is not available already (fully or partially) and recommend extending an existing dataset in favor of creating new ones. Data can be available in the bigquery-etl repository, looker-hub or in looker-spoke-default.

For modifications to schemas in restricted namespaces (see CODEOWNERS):

• [ ] Follow the change control procedure

┆Issue is synchronized with this Jira Task

[dataops-ci-bot]

Integration report for "update to schema.yaml"

sql.diff

Click to expand!

```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_search.py /tmp/workspace/generated-sql/dags/bqetl_search.py --- /tmp/workspace/main-generated-sql/dags/bqetl_search.py 2024-06-03 16:34:06.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_search.py 2024-06-03 16:22:55.000000000 +0000 @@ -52,6 +52,18 @@ tags=tags, ) as dag: + wait_for_copy_deduplicate_main_ping = ExternalTaskSensor( + task_id="wait_for_copy_deduplicate_main_ping", + external_dag_id="copy_deduplicate", + external_task_id="copy_deduplicate_main_ping", + execution_delta=datetime.timedelta(seconds=7200), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + wait_for_telemetry_derived__clients_daily_joined__v1 = ExternalTaskSensor( task_id="wait_for_telemetry_derived__clients_daily_joined__v1", external_dag_id="bqetl_main_summary", @@ -262,6 +274,10 @@ ) search_derived__search_clients_daily__v8.set_upstream( + wait_for_copy_deduplicate_main_ping + ) + + search_derived__search_clients_daily__v8.set_upstream( wait_for_telemetry_derived__clients_daily_joined__v1 ) Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix: baseline_clients_last_seen_extended_activity Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios: baseline_clients_last_seen_extended_activity diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-03 16:18:55.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.events_v1` + `moz-fx-data-shared-prod.bedrock_live.interaction_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_derived/subscriptions_map_v1/script.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_derived/subscriptions_map_v1/script.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_derived/subscriptions_map_v1/script.sql 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_derived/subscriptions_map_v1/script.sql 2024-06-03 16:18:10.000000000 +0000 @@ -131,14 +131,6 @@ 'mozilla-builder' ), ( - 'mozilla-builders-application-2024', - 'Mozilla Builder Application', - 'd48a2578-4963-4ac9-9d4e-3005106a3606', - 'd985ba67-a7ad-47a3-8358-04d223a16079', - '6fdbce2c-0110-4bd5-94ce-ff6f4e2ca905', - 'mozilla-builders-application-2024' - ), - ( 'mozilla-innovation', 'Innovation Newsletter', '39fcaf11-2926-4366-b756-2c02e65cbce7', diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- name: submission_date +- mode: NULLABLE + name: submission_date type: DATE - mode: NULLABLE -- name: source +- mode: NULLABLE + name: source type: STRING - mode: NULLABLE -- name: event_type +- mode: NULLABLE + name: event_type type: STRING - mode: NULLABLE -- name: form_factor +- mode: NULLABLE + name: form_factor type: STRING - mode: NULLABLE -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: subdivision1 +- mode: NULLABLE + name: subdivision1 type: STRING - mode: NULLABLE -- name: advertiser +- mode: NULLABLE + name: advertiser type: STRING - mode: NULLABLE -- name: release_channel +- mode: NULLABLE + name: release_channel type: STRING - mode: NULLABLE -- name: position +- mode: NULLABLE + name: position type: INTEGER - mode: NULLABLE -- name: provider +- mode: NULLABLE + name: provider type: STRING - mode: NULLABLE -- name: match_type +- mode: NULLABLE + name: match_type type: STRING - mode: NULLABLE -- name: normalized_os +- mode: NULLABLE + name: normalized_os type: STRING - mode: NULLABLE -- name: suggest_data_sharing_enabled +- mode: NULLABLE + name: suggest_data_sharing_enabled type: BOOLEAN - mode: NULLABLE -- name: event_count +- mode: NULLABLE + name: event_count type: INTEGER - mode: NULLABLE -- name: user_count +- mode: NULLABLE + name: user_count type: INTEGER - mode: NULLABLE -- name: query_type +- mode: NULLABLE + name: query_type type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- name: submission_date +- mode: NULLABLE + name: submission_date type: DATE - mode: NULLABLE -- name: form_factor +- mode: NULLABLE + name: form_factor type: STRING - mode: NULLABLE -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: advertiser +- mode: NULLABLE + name: advertiser type: STRING - mode: NULLABLE -- name: normalized_os +- mode: NULLABLE + name: normalized_os type: STRING - mode: NULLABLE -- name: release_channel +- mode: NULLABLE + name: release_channel type: STRING - mode: NULLABLE -- name: position +- mode: NULLABLE + name: position type: INTEGER - mode: NULLABLE -- name: provider +- mode: NULLABLE + name: provider type: STRING - mode: NULLABLE -- name: match_type +- mode: NULLABLE + name: match_type type: STRING - mode: NULLABLE -- name: suggest_data_sharing_enabled +- mode: NULLABLE + name: suggest_data_sharing_enabled type: BOOLEAN - mode: NULLABLE -- name: impression_count +- mode: NULLABLE + name: impression_count type: INTEGER - mode: NULLABLE -- name: click_count +- mode: NULLABLE + name: click_count type: INTEGER - mode: NULLABLE -- name: query_type +- mode: NULLABLE + name: query_type type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/baseline_clients_last_seen_extended_activity/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/baseline_clients_last_seen_extended_activity/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/baseline_clients_last_seen_extended_activity/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/baseline_clients_last_seen_extended_activity/metadata.yaml 2024-06-03 16:18:55.000000000 +0000 @@ -0,0 +1,14 @@ +friendly_name: Baseline Clients Last Seen Extended Activity +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.fenix.baseline_clients_last_seen + - moz-fx-data-shared-prod.fenix.firefox_android_clients diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/baseline_clients_last_seen_extended_activity/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/baseline_clients_last_seen_extended_activity/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/baseline_clients_last_seen_extended_activity/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/baseline_clients_last_seen_extended_activity/view.sql 2024-06-03 16:18:10.000000000 +0000 @@ -0,0 +1,50 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.fenix.baseline_clients_last_seen_extended_activity` +AS +-- This view is being added temporarily until issues preventing +-- https://github.com/mozilla/bigquery-etl/pull/5434 +-- from merging have been resolved. +SELECT + last_seen.*, + CASE + WHEN LOWER(isp) = 'browserstack' + THEN CONCAT("Fenix", ' ', isp) + WHEN LOWER(clients.distribution_id) = 'mozillaonline' + THEN CONCAT("Fenix", ' ', clients.distribution_id) + ELSE "Fenix" + END AS app_name, + -- Activity fields to support metrics built on top of activity + CASE + WHEN BIT_COUNT(days_active_bits) + BETWEEN 1 + AND 6 + THEN 'infrequent_user' + WHEN BIT_COUNT(days_active_bits) + BETWEEN 7 + AND 13 + THEN 'casual_user' + WHEN BIT_COUNT(days_active_bits) + BETWEEN 14 + AND 20 + THEN 'regular_user' + WHEN BIT_COUNT(days_active_bits) >= 21 + THEN 'core_user' + ELSE 'other' + END AS activity_segment, + IFNULL(mozfun.bits28.days_since_seen(days_active_bits) = 0, FALSE) AS is_dau, + IFNULL(mozfun.bits28.days_since_seen(days_active_bits) < 7, FALSE) AS is_wau, + IFNULL(mozfun.bits28.days_since_seen(days_active_bits) < 28, FALSE) AS is_mau, + -- Metrics based on pings sent + IFNULL(mozfun.bits28.days_since_seen(days_seen_bits) = 0, FALSE) AS is_daily_user, + IFNULL(mozfun.bits28.days_since_seen(days_seen_bits) < 7, FALSE) AS is_weekly_user, + IFNULL(mozfun.bits28.days_since_seen(days_seen_bits) < 28, FALSE) AS is_monthly_user, + ( + LOWER(IFNULL(isp, "")) <> "browserstack" + AND LOWER(IFNULL(clients.distribution_id, "")) <> "mozillaonline" + ) AS is_mobile, -- Indicates which records should be used for mobile KPI metric calculations. + FALSE AS is_desktop, +FROM + `moz-fx-data-shared-prod.fenix.baseline_clients_last_seen` AS last_seen +LEFT JOIN + `moz-fx-data-shared-prod.fenix.firefox_android_clients` AS clients + USING (client_id, first_seen_date) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -26,9 +26,6 @@ - name: adjust_network type: STRING mode: NULLABLE -- name: install_source - type: STRING - mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -48,10 +48,6 @@ description: 'The type of source of a client installation. ' -- name: install_source - type: STRING - mode: NULLABLE - description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/backfill.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/backfill.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/backfill.yaml 2024-06-03 16:31:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/backfill.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -4,4 +4,4 @@ reason: The table is created, this is to populate it with data. watchers: - kik@mozilla.com - status: Complete + status: Initiate diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-03 16:31:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-03 16:18:55.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-03 16:18:55.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -80,7 +80,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios/baseline_clients_last_seen_extended_activity/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios/baseline_clients_last_seen_extended_activity/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios/baseline_clients_last_seen_extended_activity/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios/baseline_clients_last_seen_extended_activity/metadata.yaml 2024-06-03 16:18:55.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Baseline Clients Last Seen Extended Activity +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.firefox_ios.baseline_clients_last_seen diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios/baseline_clients_last_seen_extended_activity/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios/baseline_clients_last_seen_extended_activity/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios/baseline_clients_last_seen_extended_activity/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios/baseline_clients_last_seen_extended_activity/view.sql 2024-06-03 16:18:10.000000000 +0000 @@ -0,0 +1,44 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.firefox_ios.baseline_clients_last_seen_extended_activity` +AS +-- This view is being added temporarily until issues preventing +-- https://github.com/mozilla/bigquery-etl/pull/5434 +-- from merging have been resolved. +SELECT + last_seen.*, + CASE + WHEN LOWER(isp) = 'browserstack' + THEN CONCAT("Firefox iOS", ' ', isp) + ELSE "Firefox iOS" + END AS app_name, + -- Activity fields to support metrics built on top of activity + CASE + WHEN BIT_COUNT(days_active_bits) + BETWEEN 1 + AND 6 + THEN 'infrequent_user' + WHEN BIT_COUNT(days_active_bits) + BETWEEN 7 + AND 13 + THEN 'casual_user' + WHEN BIT_COUNT(days_active_bits) + BETWEEN 14 + AND 20 + THEN 'regular_user' + WHEN BIT_COUNT(days_active_bits) >= 21 + THEN 'core_user' + ELSE 'other' + END AS activity_segment, + IFNULL(mozfun.bits28.days_since_seen(days_active_bits) = 0, FALSE) AS is_dau, + IFNULL(mozfun.bits28.days_since_seen(days_active_bits) < 7, FALSE) AS is_wau, + IFNULL(mozfun.bits28.days_since_seen(days_active_bits) < 28, FALSE) AS is_mau, + -- Metrics based on pings sent + IFNULL(mozfun.bits28.days_since_seen(days_seen_bits) = 0, FALSE) AS is_daily_user, + IFNULL(mozfun.bits28.days_since_seen(days_seen_bits) < 7, FALSE) AS is_weekly_user, + IFNULL(mozfun.bits28.days_since_seen(days_seen_bits) < 28, FALSE) AS is_monthly_user, + ( + LOWER(IFNULL(isp, "")) <> "browserstack" + ) AS is_mobile, -- Indicates which records should be used for mobile KPI metric calculations. + FALSE AS is_desktop, +FROM + `moz-fx-data-shared-prod.firefox_ios.baseline_clients_last_seen` AS last_seen diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: city +- mode: NULLABLE + name: city type: STRING - mode: NULLABLE -- name: datetime +- mode: NULLABLE + name: datetime type: TIMESTAMP - mode: NULLABLE -- name: proportion_undefined +- mode: NULLABLE + name: proportion_undefined type: FLOAT - mode: NULLABLE -- name: proportion_timeout +- mode: NULLABLE + name: proportion_timeout type: FLOAT - mode: NULLABLE -- name: proportion_abort +- mode: NULLABLE + name: proportion_abort type: FLOAT - mode: NULLABLE -- name: proportion_unreachable +- mode: NULLABLE + name: proportion_unreachable type: FLOAT - mode: NULLABLE -- name: proportion_terminated +- mode: NULLABLE + name: proportion_terminated type: FLOAT - mode: NULLABLE -- name: proportion_channel_open +- mode: NULLABLE + name: proportion_channel_open type: FLOAT - mode: NULLABLE -- name: avg_dns_success_time +- mode: NULLABLE + name: avg_dns_success_time type: FLOAT - mode: NULLABLE -- name: missing_dns_success +- mode: NULLABLE + name: missing_dns_success type: FLOAT - mode: NULLABLE -- name: avg_dns_failure_time +- mode: NULLABLE + name: avg_dns_failure_time type: FLOAT - mode: NULLABLE -- name: missing_dns_failure +- mode: NULLABLE + name: missing_dns_failure type: FLOAT - mode: NULLABLE -- name: count_dns_failure +- mode: NULLABLE + name: count_dns_failure type: FLOAT - mode: NULLABLE -- name: ssl_error_prop +- mode: NULLABLE + name: ssl_error_prop type: FLOAT - mode: NULLABLE -- name: avg_tls_handshake_time +- mode: NULLABLE + name: avg_tls_handshake_time type: FLOAT - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-03 16:18:55.000000000 +0000 @@ -45,7 +45,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -75,7 +75,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1990,7 +1990,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2000,7 +2000,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.events_v1` + `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2010,7 +2010,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2081,7 +2081,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` + `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` UNION ALL SELECT submission_timestamp, @@ -2091,7 +2091,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` + `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2162,7 +2162,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -2172,7 +2172,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -1,10 +1,7 @@ fields: -- name: build_hour - type: DATETIME - mode: NULLABLE -- name: geckoview_major_version - type: INTEGER - mode: NULLABLE -- name: n_pings - type: INTEGER - mode: NULLABLE +- type: DATETIME + name: build_hour +- type: INTEGER + name: geckoview_major_version +- type: INTEGER + name: n_pings diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/query.sql 2024-06-03 16:31:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/query.sql 2024-06-03 16:18:10.000000000 +0000 @@ -13,6 +13,7 @@ os, os_version, is_default_browser, + policies_is_enterprise, channel, CAST( NULL AS STRING @@ -47,4 +48,5 @@ os, os_version, is_default_browser, + policies_is_enterprise, channel diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/schema.yaml 2024-06-03 16:31:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -42,6 +42,9 @@ name: is_default_browser type: BOOLEAN - mode: NULLABLE + name: policies_is_enterprise + type: BOOLEAN +- mode: NULLABLE name: channel type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql 2024-06-03 16:31:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql 2024-06-03 16:18:10.000000000 +0000 @@ -51,6 +51,22 @@ client_id, submission_date ), +is_enterprise_policies AS ( + SELECT + client_id, + DATE(submission_timestamp) as submission_date, + mozfun.stats.mode_last( + ARRAY_AGG(payload.processes.parent.scalars.policies_is_enterprise ORDER BY submission_timestamp) + ) AS policies_is_enterprise + FROM + `moz-fx-data-shared-prod`.telemetry_stable.main_v5 + WHERE + normalized_app_name = 'Firefox' + AND document_id IS NOT NULL + GROUP BY + client_id, + submission_date +), combined_access_point AS ( SELECT * EXCEPT (has_adblocker_addon), @@ -105,6 +121,9 @@ LEFT JOIN clients_with_adblocker_addons USING (client_id, submission_date) + LEFT JOIN + is_enterprise_policies + USING (client_id, submission_date) ), augmented AS ( SELECT @@ -229,6 +248,7 @@ country, get_search_addon_version(active_addons) AS addon_version, has_adblocker_addon, + policies_is_enterprise, app_version, distribution_id, locale, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml 2024-06-03 16:31:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -45,6 +45,9 @@ name: is_default_browser type: BOOLEAN - mode: NULLABLE + name: policies_is_enterprise + type: BOOLEAN +- mode: NULLABLE name: profile_creation_date type: INTEGER - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -1,7 +1,9 @@ fields: - name: postal_district_code type: STRING - mode: NULLABLE + mode: REQUIRED + description: One-character Canadian postal district code. - name: province_code type: STRING mode: NULLABLE + description: Two-character Canadian province/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -1,28 +1,47 @@ fields: - name: name + description: Official country name per ISO 3166 type: STRING - mode: NULLABLE + mode: REQUIRED - name: code + description: ISO 3166 alpha-2 country code type: STRING - mode: NULLABLE + mode: REQUIRED - name: code_3 + description: ISO 3166 alpha-3 country code type: STRING - mode: NULLABLE + mode: REQUIRED - name: region_name + description: Region name. These are based on the UN Statistics Division standard + country or area codes for statistical use (M49), but with the "Americas" region + split into "North America" and "South America". type: STRING - mode: NULLABLE + mode: REQUIRED - name: subregion_name + description: Sub-region name. These are based on UN Statistics Division standard + country or area codes for statistical use (M49), but with the "Latin America and the + Caribbean" and "Sub-Saharan Africa" sub-regions split into more specific + sub-regions. type: STRING - mode: NULLABLE + mode: REQUIRED - name: pocket_available_on_newtab - type: BOOLEAN - mode: NULLABLE + description: Whether Pocket is available on the newtab page in this country. Note + that Pocket might only be available in certain locales/languages within a country. + type: BOOL + mode: REQUIRED - name: mozilla_vpn_available - type: BOOLEAN - mode: NULLABLE + description: Whether Mozilla VPN is available in this country. + type: BOOL + mode: REQUIRED - name: sponsored_tiles_available_on_newtab - type: BOOLEAN - mode: NULLABLE + description: Whether sponsored tiles are available on the newtab page in this country. + Note that Pocket might only be available in certain locales/languages within a + country. + type: BOOL + mode: REQUIRED - name: ads_value_tier + description: Lowercase label detailing the monetary value tier that Mozilla Ads + assign to that region based on market size and our existing products, e.g., tier + 1, tier 2, etc. type: STRING - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -1,7 +1,10 @@ fields: - name: name + description: An alias for a country's name (including misspellings and alternate + encodings). type: STRING - mode: NULLABLE + mode: REQUIRED - name: code + description: ISO 3166 alpha-2 country code type: STRING - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: incident +- mode: NULLABLE + name: incident type: STRING - mode: NULLABLE -- name: description +- mode: NULLABLE + name: description type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE -- name: product +- mode: NULLABLE + name: product type: STRING - mode: NULLABLE -- name: version +- mode: NULLABLE + name: version type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -1,16 +1,27 @@ fields: -- name: value +- mode: NULLABLE + description: Hex value assigned to the TLS cipher, in format like "0x00,0x84"; note + some values are ranges or contain wildcards + name: value type: STRING - mode: NULLABLE -- name: description +- mode: NULLABLE + description: Human-readable name of the TLS cipher + name: description type: STRING - mode: NULLABLE -- name: dtls_ok +- mode: NULLABLE + description: Any TLS cipher suite that is specified for use with DTLS MUST define + limits on the use of the associated AEAD function that preserves margins for both + confidentiality and integrity, as specified in [RFC-ietf-tls-dtls13-43] + name: dtls_ok type: BOOLEAN - mode: NULLABLE -- name: recommended +- mode: NULLABLE + description: Whether the TLS cipher is recommended by the IETF. If an item is not + marked as "recommended", it does not necessarily mean that it is flawed; rather, + it indicates that the item either has not been through the IETF consensus process, + has limited applicability, or is intended only for specific use cases + name: recommended type: BOOLEAN - mode: NULLABLE -- name: reference +- mode: NULLABLE + description: RFCs or associated reference material for the TLS cipher + name: reference type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -1,13 +1,17 @@ fields: - name: code_3 + description: ISO 639 alpha-3 language code. type: STRING - mode: NULLABLE + mode: REQUIRED - name: code_2 + description: ISO 639 alpha-2 language code (if any). type: STRING mode: NULLABLE - name: name + description: Language name. type: STRING - mode: NULLABLE + mode: REQUIRED - name: other_names + description: Other names for the language (if any). type: STRING mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -1,19 +1,19 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -1,25 +1,25 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: document_version +- mode: NULLABLE + name: document_version type: STRING - mode: NULLABLE -- name: path +- mode: NULLABLE + name: path type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: document_version +- mode: NULLABLE + name: document_version type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: path +- mode: NULLABLE + name: path type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -1,7 +1,7 @@ fields: - name: score type: NUMERIC - mode: NULLABLE + mode: REQUIRED - name: value type: NUMERIC - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-03 16:18:10.000000000 +0000 @@ -1,7 +1,9 @@ fields: - name: zip_code_prefix type: STRING - mode: NULLABLE + mode: REQUIRED + description: Three-digit US ZIP code prefix. - name: state_code type: STRING mode: NULLABLE + description: Two-character US state/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml 2024-06-03 16:18:55.000000000 +0000 @@ -2,21 +2,18 @@ - name: date type: DATE mode: NULLABLE - description: null - name: product type: STRING mode: NULLABLE - name: category type: STRING mode: NULLABLE - description: null - name: channel type: STRING mode: NULLABLE - name: build_number type: INTEGER mode: NULLABLE - description: null - name: release_date type: DATE mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-03 16:31:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-03 16:18:55.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_live.video_index_v1` + `moz-fx-data-shared-prod.viu_politica_live.main_events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_live.main_events_v1` + `moz-fx-data-shared-prod.viu_politica_live.video_index_v1` ) CROSS JOIN UNNEST(events) AS event, ```

Link to full diff

[dataops-ci-bot]

Integration report for "Fix CI issues"

sql.diff

Click to expand!

```diff Only in /tmp/workspace/main-generated-sql/dags/: bqetl_mobile_kpi_metrics.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_analytics_tables.py /tmp/workspace/generated-sql/dags/bqetl_analytics_tables.py --- /tmp/workspace/main-generated-sql/dags/bqetl_analytics_tables.py 2024-06-04 17:48:24.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_analytics_tables.py 2024-06-04 18:00:27.000000000 +0000 @@ -265,13 +265,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_feature_usage__wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", external_dag_id="bqetl_mobile_feature_usage", external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_braze.py /tmp/workspace/generated-sql/dags/bqetl_braze.py --- /tmp/workspace/main-generated-sql/dags/bqetl_braze.py 2024-06-04 17:48:24.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_braze.py 2024-06-04 18:00:28.000000000 +0000 @@ -149,7 +149,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__changed_newsletters_sync__v1 = bigquery_etl_query( @@ -162,7 +161,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__changed_products_sync__v1 = bigquery_etl_query( @@ -175,7 +173,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__changed_users__v1 = bigquery_etl_query( @@ -200,7 +197,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__changed_waitlists_sync__v1 = bigquery_etl_query( @@ -213,7 +209,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__delete_users_sync__v1 = bigquery_etl_query( @@ -226,7 +221,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__users_previous_day_snapshot__v1 = bigquery_etl_query( @@ -354,7 +348,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -368,7 +361,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -382,7 +374,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -409,7 +400,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -423,7 +413,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -437,7 +426,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_desktop_retention_model.py /tmp/workspace/generated-sql/dags/bqetl_desktop_retention_model.py --- /tmp/workspace/main-generated-sql/dags/bqetl_desktop_retention_model.py 2024-06-04 17:48:24.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_desktop_retention_model.py 2024-06-04 18:00:28.000000000 +0000 @@ -90,15 +90,13 @@ telemetry_derived__desktop_retention__v1 = bigquery_etl_query( task_id="telemetry_derived__desktop_retention__v1", - destination_table='desktop_retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', + destination_table="desktop_retention_v1", dataset_id="telemetry_derived", project_id="moz-fx-data-shared-prod", owner="mhirose@mozilla.com", email=["mhirose@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, + date_partition_parameter="submission_date", depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], ) telemetry_derived__desktop_retention_clients__v1 = bigquery_etl_query( diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_firefox_ios.py /tmp/workspace/generated-sql/dags/bqetl_firefox_ios.py --- /tmp/workspace/main-generated-sql/dags/bqetl_firefox_ios.py 2024-06-04 17:48:24.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_firefox_ios.py 2024-06-04 18:00:25.000000000 +0000 @@ -311,13 +311,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=57600)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_feature_usage__wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", external_dag_id="bqetl_mobile_feature_usage", external_task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_glean_usage.py /tmp/workspace/generated-sql/dags/bqetl_glean_usage.py --- /tmp/workspace/main-generated-sql/dags/bqetl_glean_usage.py 2024-06-04 17:48:24.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_glean_usage.py 2024-06-04 18:00:31.000000000 +0000 @@ -365,6 +365,20 @@ task_group=task_group_burnham, ) + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( + task_id="checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1", + source_table="baseline_clients_last_seen_v1", + dataset_id="accounts_cirrus_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=True, + owner="ascholtz@mozilla.com", + email=["ascholtz@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_accounts_cirrus, + ) + checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -461,13 +475,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", @@ -524,13 +531,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", @@ -587,13 +587,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", @@ -650,13 +643,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", @@ -713,13 +699,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", @@ -764,21 +743,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1_external", - parent_group=task_group_focus_android, - ) as checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1_external.set_upstream( - checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 - ) - checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -798,13 +762,6 @@ parent_group=task_group_focus_android, ) as checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1_external: ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_nondesktop__wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_nondesktop", external_task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", @@ -836,21 +793,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1_external", - parent_group=task_group_focus_android, - ) as checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1_external.set_upstream( - checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 - ) - checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -877,13 +819,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_activation__wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_mobile_activation", external_task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", @@ -934,13 +869,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_activation__wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_mobile_activation", external_task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", @@ -977,13 +905,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_activation__wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_mobile_activation", external_task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", @@ -1013,13 +934,6 @@ parent_group=task_group_focus_ios, ) as checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1_external: ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_nondesktop__wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_nondesktop", external_task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", @@ -1056,13 +970,6 @@ parent_group=task_group_klar_ios, ) as checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1_external: ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_nondesktop__wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_nondesktop", external_task_id="wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", @@ -1135,13 +1042,6 @@ parent_group=task_group_klar_android, ) as checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1_external: ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_nondesktop__wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_nondesktop", external_task_id="wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", @@ -1259,6 +1159,20 @@ task_group=task_group_pine, ) + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( + task_id="checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1", + source_table="baseline_clients_last_seen_v1", + dataset_id="accounts_cirrus_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=False, + owner="ascholtz@mozilla.com", + email=["ascholtz@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_accounts_cirrus, + ) + checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -3192,13 +3106,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", @@ -3281,13 +3188,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", @@ -3372,13 +3272,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", @@ -3463,13 +3356,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", @@ -3556,13 +3442,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", @@ -3741,21 +3620,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "org_mozilla_focus_beta_derived__baseline_clients_daily__v1_external", - parent_group=task_group_focus_android, - ) as org_mozilla_focus_beta_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_focus_beta_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_focus_beta_derived__baseline_clients_daily__v1 - ) - org_mozilla_focus_beta_derived__baseline_clients_first_seen__v1 = ( bigquery_etl_query( task_id="org_mozilla_focus_beta_derived__baseline_clients_first_seen__v1", @@ -3813,21 +3677,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "org_mozilla_focus_derived__baseline_clients_daily__v1_external", - parent_group=task_group_focus_android, - ) as org_mozilla_focus_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_focus_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_focus_derived__baseline_clients_daily__v1 - ) - org_mozilla_focus_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_focus_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -3883,21 +3732,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "org_mozilla_focus_nightly_derived__baseline_clients_daily__v1_external", - parent_group=task_group_focus_android, - ) as org_mozilla_focus_nightly_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_focus_nightly_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_focus_nightly_derived__baseline_clients_daily__v1 - ) - org_mozilla_focus_nightly_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_focus_nightly_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -3973,13 +3807,6 @@ execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", ) - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - org_mozilla_ios_fennec_derived__baseline_clients_daily__v1_external.set_upstream( org_mozilla_ios_fennec_derived__baseline_clients_daily__v1 ) @@ -4059,13 +3886,6 @@ execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", ) - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - org_mozilla_ios_firefox_derived__baseline_clients_daily__v1_external.set_upstream( org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 ) @@ -4149,13 +3969,6 @@ execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", ) - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1_external.set_upstream( org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 ) @@ -4325,21 +4138,6 @@ task_group=task_group_focus_ios, ) - with TaskGroup( - "org_mozilla_ios_focus_derived__baseline_clients_daily__v1_external", - parent_group=task_group_focus_ios, - ) as org_mozilla_ios_focus_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_ios_focus_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_ios_focus_derived__baseline_clients_daily__v1 - ) - org_mozilla_ios_focus_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_ios_focus_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -4395,21 +4193,6 @@ task_group=task_group_klar_ios, ) - with TaskGroup( - "org_mozilla_ios_klar_derived__baseline_clients_daily__v1_external", - parent_group=task_group_klar_ios, - ) as org_mozilla_ios_klar_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_ios_klar_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_ios_klar_derived__baseline_clients_daily__v1 - ) - org_mozilla_ios_klar_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_ios_klar_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -4524,21 +4307,6 @@ task_group=task_group_klar_android, ) - with TaskGroup( - "org_mozilla_klar_derived__baseline_clients_daily__v1_external", - parent_group=task_group_klar_android, - ) as org_mozilla_klar_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_klar_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_klar_derived__baseline_clients_daily__v1 - ) - org_mozilla_klar_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_klar_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -5100,6 +4868,18 @@ burnham_derived__metrics_clients_daily__v1 ) + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + accounts_cirrus_derived__baseline_clients_last_seen__v1 + ) + + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 + ) + checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1.set_upstream( checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 ) @@ -5384,6 +5164,18 @@ pine_derived__baseline_clients_last_seen__v1 ) + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + accounts_cirrus_derived__baseline_clients_last_seen__v1 + ) + + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 + ) + checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1.set_upstream( checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_mobile_kpi_metrics.py /tmp/workspace/generated-sql/dags/bqetl_mobile_kpi_metrics.py --- /tmp/workspace/main-generated-sql/dags/bqetl_mobile_kpi_metrics.py 2024-06-04 17:48:24.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_mobile_kpi_metrics.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,765 +0,0 @@ -# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py - -from airflow import DAG -from airflow.sensors.external_task import ExternalTaskMarker -from airflow.sensors.external_task import ExternalTaskSensor -from airflow.utils.task_group import TaskGroup -import datetime -from operators.gcp_container_operator import GKEPodOperator -from utils.constants import ALLOWED_STATES, FAILED_STATES -from utils.gcp import bigquery_etl_query, bigquery_dq_check - -docs = """ -### bqetl_mobile_kpi_metrics - -Built from bigquery-etl repo, [`dags/bqetl_mobile_kpi_metrics.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_mobile_kpi_metrics.py) - -#### Description - -Generates support metrics for mobile KPI's -#### Owner - -kik@mozilla.com - -#### Tags - -* impact/tier_1 -* repo/bigquery-etl -""" - - -default_args = { - "owner": "kik@mozilla.com", - "start_date": datetime.datetime(2024, 6, 3, 0, 0), - "end_date": None, - "email": ["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - "depends_on_past": False, - "retry_delay": datetime.timedelta(seconds=1800), - "email_on_failure": True, - "email_on_retry": False, - "retries": 1, -} - -tags = ["impact/tier_1", "repo/bigquery-etl"] - -with DAG( - "bqetl_mobile_kpi_metrics", - default_args=default_args, - schedule_interval="0 12 * * *", - doc_md=docs, - tags=tags, -) as dag: - - task_group_fenix = TaskGroup("fenix") - - task_group_firefox_ios = TaskGroup("firefox_ios") - - task_group_focus_android = TaskGroup("focus_android") - - task_group_focus_ios = TaskGroup("focus_ios") - - task_group_klar_android = TaskGroup("klar_android") - - task_group_klar_ios = TaskGroup("klar_ios") - - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 = ( - ExternalTaskSensor( - task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - external_dag_id="bqetl_analytics_tables", - external_task_id="checks__fail_fenix_derived__firefox_android_clients__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - ) - - wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fenix_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_firefox_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - external_dag_id="bqetl_firefox_ios", - external_task_id="checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - execution_delta=datetime.timedelta(seconds=28800), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_ios.checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_ios.org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_android.checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_android.org_mozilla_klar_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_ios.checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_ios.org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - fenix_derived__engagement__v1 = bigquery_etl_query( - task_id="fenix_derived__engagement__v1", - destination_table="engagement_v1", - dataset_id="fenix_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_fenix, - ) - - fenix_derived__retention__v1 = bigquery_etl_query( - task_id="fenix_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="fenix_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_fenix, - ) - - firefox_ios_derived__engagement__v1 = bigquery_etl_query( - task_id="firefox_ios_derived__engagement__v1", - destination_table="engagement_v1", - dataset_id="firefox_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_firefox_ios, - ) - - firefox_ios_derived__retention__v1 = bigquery_etl_query( - task_id="firefox_ios_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="firefox_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_firefox_ios, - ) - - focus_android_derived__engagement__v1 = bigquery_etl_query( - task_id="focus_android_derived__engagement__v1", - destination_table="engagement_v1", - dataset_id="focus_android_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_focus_android, - ) - - focus_android_derived__retention__v1 = bigquery_etl_query( - task_id="focus_android_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="focus_android_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_focus_android, - ) - - focus_ios_derived__engagement__v1 = bigquery_etl_query( - task_id="focus_ios_derived__engagement__v1", - destination_table="engagement_v1", - dataset_id="focus_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_focus_ios, - ) - - focus_ios_derived__retention__v1 = bigquery_etl_query( - task_id="focus_ios_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="focus_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_focus_ios, - ) - - klar_android_derived__engagement__v1 = bigquery_etl_query( - task_id="klar_android_derived__engagement__v1", - destination_table="engagement_v1", - dataset_id="klar_android_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_klar_android, - ) - - klar_android_derived__retention__v1 = bigquery_etl_query( - task_id="klar_android_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="klar_android_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_klar_android, - ) - - klar_ios_derived__engagement__v1 = bigquery_etl_query( - task_id="klar_ios_derived__engagement__v1", - destination_table="engagement_v1", - dataset_id="klar_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_klar_ios, - ) - - klar_ios_derived__retention__v1 = bigquery_etl_query( - task_id="klar_ios_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="klar_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_klar_ios, - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fenne ```

⚠️ Only part of the diff is displayed.

Link to full diff

[dataops-ci-bot]

Integration report for "Merge branch 'main' into RS_1233_add_is_enterprise_policies_to_search"

sql.diff

Click to expand!

```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_search.py /tmp/workspace/generated-sql/dags/bqetl_search.py --- /tmp/workspace/main-generated-sql/dags/bqetl_search.py 2024-06-04 18:14:46.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_search.py 2024-06-04 18:16:47.000000000 +0000 @@ -52,6 +52,18 @@ tags=tags, ) as dag: + wait_for_copy_deduplicate_main_ping = ExternalTaskSensor( + task_id="wait_for_copy_deduplicate_main_ping", + external_dag_id="copy_deduplicate", + external_task_id="copy_deduplicate_main_ping", + execution_delta=datetime.timedelta(seconds=7200), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + wait_for_telemetry_derived__clients_daily_joined__v1 = ExternalTaskSensor( task_id="wait_for_telemetry_derived__clients_daily_joined__v1", external_dag_id="bqetl_main_summary", @@ -262,6 +274,10 @@ ) search_derived__search_clients_daily__v8.set_upstream( + wait_for_copy_deduplicate_main_ping + ) + + search_derived__search_clients_daily__v8.set_upstream( wait_for_telemetry_derived__clients_daily_joined__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/query.sql 2024-06-04 18:11:17.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/query.sql 2024-06-04 18:11:15.000000000 +0000 @@ -13,6 +13,7 @@ os, os_version, is_default_browser, + policies_is_enterprise, channel, CAST( NULL AS STRING @@ -47,4 +48,5 @@ os, os_version, is_default_browser, + policies_is_enterprise, channel diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/schema.yaml 2024-06-04 18:11:17.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/schema.yaml 2024-06-04 18:11:15.000000000 +0000 @@ -42,6 +42,9 @@ name: is_default_browser type: BOOLEAN - mode: NULLABLE + name: policies_is_enterprise + type: BOOLEAN +- mode: NULLABLE name: channel type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql 2024-06-04 18:11:17.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql 2024-06-04 18:11:15.000000000 +0000 @@ -51,6 +51,26 @@ client_id, submission_date ), +is_enterprise_policies AS ( + SELECT + client_id, + DATE(submission_timestamp) AS submission_date, + mozfun.stats.mode_last( + ARRAY_AGG( + payload.processes.parent.scalars.policies_is_enterprise + ORDER BY + submission_timestamp + ) + ) AS policies_is_enterprise + FROM + `moz-fx-data-shared-prod`.telemetry_stable.main_v5 + WHERE + normalized_app_name = 'Firefox' + AND document_id IS NOT NULL + GROUP BY + client_id, + submission_date +), combined_access_point AS ( SELECT * EXCEPT (has_adblocker_addon), @@ -105,6 +125,9 @@ LEFT JOIN clients_with_adblocker_addons USING (client_id, submission_date) + LEFT JOIN + is_enterprise_policies + USING (client_id, submission_date) ), augmented AS ( SELECT @@ -229,6 +252,7 @@ country, get_search_addon_version(active_addons) AS addon_version, has_adblocker_addon, + policies_is_enterprise, app_version, distribution_id, locale, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml 2024-06-04 18:11:17.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml 2024-06-04 18:11:15.000000000 +0000 @@ -45,6 +45,9 @@ name: is_default_browser type: BOOLEAN - mode: NULLABLE + name: policies_is_enterprise + type: BOOLEAN +- mode: NULLABLE name: profile_creation_date type: INTEGER - mode: NULLABLE ```

Link to full diff

[dataops-ci-bot]

Integration report for "Merge branch 'main' into RS_1233_add_is_enterprise_policies_to_search"

sql.diff

Click to expand!

```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_search.py /tmp/workspace/generated-sql/dags/bqetl_search.py --- /tmp/workspace/main-generated-sql/dags/bqetl_search.py 2024-06-04 19:18:16.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_search.py 2024-06-04 19:19:23.000000000 +0000 @@ -52,6 +52,18 @@ tags=tags, ) as dag: + wait_for_copy_deduplicate_main_ping = ExternalTaskSensor( + task_id="wait_for_copy_deduplicate_main_ping", + external_dag_id="copy_deduplicate", + external_task_id="copy_deduplicate_main_ping", + execution_delta=datetime.timedelta(seconds=7200), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + wait_for_telemetry_derived__clients_daily_joined__v1 = ExternalTaskSensor( task_id="wait_for_telemetry_derived__clients_daily_joined__v1", external_dag_id="bqetl_main_summary", @@ -262,6 +274,10 @@ ) search_derived__search_clients_daily__v8.set_upstream( + wait_for_copy_deduplicate_main_ping + ) + + search_derived__search_clients_daily__v8.set_upstream( wait_for_telemetry_derived__clients_daily_joined__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/query.sql 2024-06-04 19:14:51.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/query.sql 2024-06-04 19:14:42.000000000 +0000 @@ -13,6 +13,7 @@ os, os_version, is_default_browser, + policies_is_enterprise, channel, CAST( NULL AS STRING @@ -47,4 +48,5 @@ os, os_version, is_default_browser, + policies_is_enterprise, channel diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/schema.yaml 2024-06-04 19:14:51.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/schema.yaml 2024-06-04 19:14:42.000000000 +0000 @@ -42,6 +42,9 @@ name: is_default_browser type: BOOLEAN - mode: NULLABLE + name: policies_is_enterprise + type: BOOLEAN +- mode: NULLABLE name: channel type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql 2024-06-04 19:14:51.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql 2024-06-04 19:14:42.000000000 +0000 @@ -51,6 +51,26 @@ client_id, submission_date ), +is_enterprise_policies AS ( + SELECT + client_id, + DATE(submission_timestamp) AS submission_date, + mozfun.stats.mode_last( + ARRAY_AGG( + payload.processes.parent.scalars.policies_is_enterprise + ORDER BY + submission_timestamp + ) + ) AS policies_is_enterprise + FROM + `moz-fx-data-shared-prod`.telemetry_stable.main_v5 + WHERE + normalized_app_name = 'Firefox' + AND document_id IS NOT NULL + GROUP BY + client_id, + submission_date +), combined_access_point AS ( SELECT * EXCEPT (has_adblocker_addon), @@ -105,6 +125,9 @@ LEFT JOIN clients_with_adblocker_addons USING (client_id, submission_date) + LEFT JOIN + is_enterprise_policies + USING (client_id, submission_date) ), augmented AS ( SELECT @@ -229,6 +252,7 @@ country, get_search_addon_version(active_addons) AS addon_version, has_adblocker_addon, + policies_is_enterprise, app_version, distribution_id, locale, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml 2024-06-04 19:14:51.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml 2024-06-04 19:14:42.000000000 +0000 @@ -45,6 +45,9 @@ name: is_default_browser type: BOOLEAN - mode: NULLABLE + name: policies_is_enterprise + type: BOOLEAN +- mode: NULLABLE name: profile_creation_date type: INTEGER - mode: NULLABLE ```

Link to full diff

[dataops-ci-bot]

Integration report for "Fix the tests"

sql.diff

Click to expand!

```diff Only in /tmp/workspace/main-generated-sql/dags/: bqetl_desktop_conv_evnt_categorization.py Only in /tmp/workspace/main-generated-sql/dags/: bqetl_mobile_kpi_metrics.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_analytics_tables.py /tmp/workspace/generated-sql/dags/bqetl_analytics_tables.py --- /tmp/workspace/main-generated-sql/dags/bqetl_analytics_tables.py 2024-06-05 13:39:08.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_analytics_tables.py 2024-06-05 13:51:22.000000000 +0000 @@ -265,13 +265,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_feature_usage__wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", external_dag_id="bqetl_mobile_feature_usage", external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", @@ -324,13 +317,6 @@ ) ExternalTaskMarker( - task_id="bqetl_desktop_conv_evnt_categorization__wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", - external_dag_id="bqetl_desktop_conv_evnt_categorization", - external_task_id="wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mozilla_org_derived__wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", external_dag_id="bqetl_mozilla_org_derived", external_task_id="wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_braze.py /tmp/workspace/generated-sql/dags/bqetl_braze.py --- /tmp/workspace/main-generated-sql/dags/bqetl_braze.py 2024-06-05 13:39:08.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_braze.py 2024-06-05 13:51:22.000000000 +0000 @@ -149,7 +149,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__changed_newsletters_sync__v1 = bigquery_etl_query( @@ -162,7 +161,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__changed_products_sync__v1 = bigquery_etl_query( @@ -175,7 +173,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__changed_users__v1 = bigquery_etl_query( @@ -200,7 +197,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__changed_waitlists_sync__v1 = bigquery_etl_query( @@ -213,7 +209,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__delete_users_sync__v1 = bigquery_etl_query( @@ -226,7 +221,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__users_previous_day_snapshot__v1 = bigquery_etl_query( @@ -354,7 +348,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -368,7 +361,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -382,7 +374,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -409,7 +400,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -423,7 +413,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -437,7 +426,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_desktop_conv_evnt_categorization.py /tmp/workspace/generated-sql/dags/bqetl_desktop_conv_evnt_categorization.py --- /tmp/workspace/main-generated-sql/dags/bqetl_desktop_conv_evnt_categorization.py 2024-06-05 13:39:08.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_desktop_conv_evnt_categorization.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,116 +0,0 @@ -# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py - -from airflow import DAG -from airflow.sensors.external_task import ExternalTaskMarker -from airflow.sensors.external_task import ExternalTaskSensor -from airflow.utils.task_group import TaskGroup -import datetime -from operators.gcp_container_operator import GKEPodOperator -from utils.constants import ALLOWED_STATES, FAILED_STATES -from utils.gcp import bigquery_etl_query, bigquery_dq_check - -docs = """ -### bqetl_desktop_conv_evnt_categorization - -Built from bigquery-etl repo, [`dags/bqetl_desktop_conv_evnt_categorization.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_desktop_conv_evnt_categorization.py) - -#### Description - -Loads the desktop conversion event tables -#### Owner - -kwindau@mozilla.com - -#### Tags - -* impact/tier_2 -* repo/bigquery-etl -""" - - -default_args = { - "owner": "kwindau@mozilla.com", - "start_date": datetime.datetime(2024, 6, 4, 0, 0), - "end_date": None, - "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], - "depends_on_past": False, - "retry_delay": datetime.timedelta(seconds=1800), - "email_on_failure": True, - "email_on_retry": False, - "retries": 2, -} - -tags = ["impact/tier_2", "repo/bigquery-etl"] - -with DAG( - "bqetl_desktop_conv_evnt_categorization", - default_args=default_args, - schedule_interval="0 12 * * *", - doc_md=docs, - tags=tags, -) as dag: - - wait_for_checks__fail_telemetry_derived__clients_first_seen__v2 = ( - ExternalTaskSensor( - task_id="wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", - external_dag_id="bqetl_analytics_tables", - external_task_id="checks__fail_telemetry_derived__clients_first_seen__v2", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - ) - - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 = ExternalTaskSensor( - task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - external_dag_id="bqetl_main_summary", - external_task_id="checks__fail_telemetry_derived__clients_last_seen__v2", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - checks__warn_google_ads_derived__conversion_event_categorization__v1 = bigquery_dq_check( - task_id="checks__warn_google_ads_derived__conversion_event_categorization__v1", - source_table='conversion_event_categorization_v1${{ macros.ds_format(macros.ds_add(ds, -14), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="google_ads_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], - depends_on_past=False, - parameters=["report_date:DATE:{{macros.ds_add(ds, -14)}}"] - + ["submission_date:DATE:{{ds}}"], - retries=0, - ) - - google_ads_derived__conversion_event_categorization__v1 = bigquery_etl_query( - task_id="google_ads_derived__conversion_event_categorization__v1", - destination_table='conversion_event_categorization_v1${{ macros.ds_format(macros.ds_add(ds, -14), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="google_ads_derived", - project_id="moz-fx-data-shared-prod", - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["report_date:DATE:{{macros.ds_add(ds, -14)}}"] - + ["submission_date:DATE:{{ds}}"], - ) - - checks__warn_google_ads_derived__conversion_event_categorization__v1.set_upstream( - google_ads_derived__conversion_event_categorization__v1 - ) - - google_ads_derived__conversion_event_categorization__v1.set_upstream( - wait_for_checks__fail_telemetry_derived__clients_first_seen__v2 - ) - - google_ads_derived__conversion_event_categorization__v1.set_upstream( - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 - ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_desktop_retention_model.py /tmp/workspace/generated-sql/dags/bqetl_desktop_retention_model.py --- /tmp/workspace/main-generated-sql/dags/bqetl_desktop_retention_model.py 2024-06-05 13:39:08.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_desktop_retention_model.py 2024-06-05 13:51:23.000000000 +0000 @@ -90,15 +90,13 @@ telemetry_derived__desktop_retention__v1 = bigquery_etl_query( task_id="telemetry_derived__desktop_retention__v1", - destination_table='desktop_retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', + destination_table="desktop_retention_v1", dataset_id="telemetry_derived", project_id="moz-fx-data-shared-prod", owner="mhirose@mozilla.com", email=["mhirose@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, + date_partition_parameter="submission_date", depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], ) telemetry_derived__desktop_retention_clients__v1 = bigquery_etl_query( diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_firefox_ios.py /tmp/workspace/generated-sql/dags/bqetl_firefox_ios.py --- /tmp/workspace/main-generated-sql/dags/bqetl_firefox_ios.py 2024-06-05 13:39:08.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_firefox_ios.py 2024-06-05 13:51:21.000000000 +0000 @@ -311,13 +311,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=57600)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_feature_usage__wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", external_dag_id="bqetl_mobile_feature_usage", external_task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_glean_usage.py /tmp/workspace/generated-sql/dags/bqetl_glean_usage.py --- /tmp/workspace/main-generated-sql/dags/bqetl_glean_usage.py 2024-06-05 13:39:08.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_glean_usage.py 2024-06-05 13:51:25.000000000 +0000 @@ -365,6 +365,20 @@ task_group=task_group_burnham, ) + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( + task_id="checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1", + source_table="baseline_clients_last_seen_v1", + dataset_id="accounts_cirrus_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=True, + owner="ascholtz@mozilla.com", + email=["ascholtz@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_accounts_cirrus, + ) + checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -461,13 +475,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", @@ -524,13 +531,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", @@ -587,13 +587,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", @@ -650,13 +643,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", @@ -713,13 +699,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", @@ -764,21 +743,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1_external", - parent_group=task_group_focus_android, - ) as checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1_external.set_upstream( - checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 - ) - checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -798,13 +762,6 @@ parent_group=task_group_focus_android, ) as checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1_external: ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_nondesktop__wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_nondesktop", external_task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", @@ -836,21 +793,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1_external", - parent_group=task_group_focus_android, - ) as checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1_external.set_upstream( - checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 - ) - checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -877,13 +819,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_activation__wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_mobile_activation", external_task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", @@ -934,13 +869,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_activation__wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_mobile_activation", external_task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", @@ -977,13 +905,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_activation__wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_mobile_activation", external_task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", @@ -1013,13 +934,6 @@ parent_group=task_group_focus_ios, ) as checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1_external: ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_nondesktop__wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_nondesktop", external_task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", @@ -1056,13 +970,6 @@ parent_group=task_group_klar_ios, ) as checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1_external: ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_nondesktop__wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_nondesktop", external_task_id="wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", @@ -1135,13 +1042,6 @@ parent_group=task_group_klar_android, ) as checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1_external: ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_nondesktop__wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_nondesktop", external_task_id="wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", @@ -1259,6 +1159,20 @@ task_group=task_group_pine, ) + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( + task_id="checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1", + source_table="baseline_clients_last_seen_v1", + dataset_id="accounts_cirrus_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=False, + owner="ascholtz@mozilla.com", + email=["ascholtz@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_accounts_cirrus, + ) + checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -3192,13 +3106,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", @@ -3281,13 +3188,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", @@ -3372,13 +3272,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", @@ -3463,13 +3356,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", @@ -3556,13 +3442,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", @@ -3741,21 +3620,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "org_mozilla_focus_beta_derived__baseline_clients_daily__v1_external", - parent_group=task_group_focus_android, - ) as org_mozilla_focus_beta_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_focus_beta_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_focus_beta_derived__baseline_clients_daily__v1 - ) - org_mozilla_focus_beta_derived__baseline_clients_first_seen__v1 = ( bigquery_etl_query( task_id="org_mozilla_focus_beta_derived__baseline_clients_first_seen__v1", @@ -3813,21 +3677,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "org_mozilla_focus_derived__baseline_clients_daily__v1_external", - parent_group=task_group_focus_android, - ) as org_mozilla_focus_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_focus_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_focus_derived__baseline_clients_daily__v1 - ) - org_mozilla_focus_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_focus_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -3883,21 +3732,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "org_mozilla_focus_nightly_derived__baseline_clients_daily__v1_external", - parent_group=task_group_focus_android, - ) as org_mozilla_focus_nightly_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_focus_nightly_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_focus_nightly_derived__baseline_clients_daily__v1 - ) - org_mozilla_focus_nightly_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_focus_nightly_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -3973,13 +3807,6 @@ execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", ) - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - org_mozilla_ios_fennec_derived__baseline_clients_daily__v1_external.set_upstream( org_mozilla_ios_fennec_derived__baseline_clients_daily__v1 ) @@ -4059,13 +3886,6 @@ execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", ) - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - org_mozilla_ios_firefox_derived__baseline_clients_daily__v1_external.set_upstream( org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 ) @@ -4149,13 +3969,6 @@ execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", ) - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1_external.set_upstream( org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 ) @@ -4325,21 +4138,6 @@ task_group=task_group_focus_ios, ) - with TaskGroup( - "org_mozilla_ios_focus_derived__baseline_clients_daily__v1_external", - parent_group=task_group_focus_ios, - ) as org_mozilla_ios_focus_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_ios_focus_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_ios_focus_derived__baseline_clients_daily__v1 - ) - org_mozilla_ios_focus_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_ios_focus_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -4395,21 +4193,6 @@ task_group=task_group_klar_ios, ) - with TaskGroup( - "org_mozilla_ios_klar_derived__baseline_clients_daily__v1_external", - parent_group=task_group_klar_ios, - ) as org_mozilla_ios_klar_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_ios_klar_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_ios_klar_derived__baseline_clients_daily__v1 - ) - org_mozilla_ios_klar_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_ios_klar_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -4524,21 +4307,6 @@ task_group=task_group_klar_android, ) - with TaskGroup( - "org_mozilla_klar_derived__baseline_clients_daily__v1_external", - parent_group=task_group_klar_android, - ) as org_mozilla_klar_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_klar_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_klar_derived__baseline_clients_daily__v1 - ) - org_mozilla_klar_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_klar_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -5100,6 +4868,18 @@ burnham_derived__metrics_clients_daily__v1 ) + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + accounts_cirrus_derived__baseline_clients_last_seen__v1 + ) + + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 + ) + checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1.set_upstream( checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 ) @@ -5384,6 +5164,18 @@ pine_derived__baseline_clients_last_seen__v1 ) + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + accounts_cirrus_derived__baseline_clients_last_seen__v1 + ) + + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 + ) + checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1.set_upstream( checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_main_summary.py /tmp/workspace/generated-sql/dags/bqetl_main_summary.py --- /tmp/workspace/main-generated-sql/dags/bqetl_main_summary.py 2024-06-05 13:39:07.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_main_summary.py 2024-06-05 13:51:20.000000000 +0000 @@ -144,13 +144,6 @@ ) ExternalTaskMarker( - task_id="bqetl_desktop_conv_evnt_categorization__wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - external_dag_id="bqetl_desktop_conv_evnt_categorization", - external_task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_search_dashboard__wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", external_dag_id="bqetl_search_dashboard", external_task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_mobile_kpi_metrics.py /tmp/workspace/generated-sql/dags/bqetl_mobile_kpi_metrics.py --- /tmp/workspace/main-generated-sql/dags/bqetl_mobile_kpi_metrics.py 2024-06-05 13:39:08.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_mobile_kpi_metrics.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,765 +0,0 @@ -# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py - -from airflow import DAG -from airflow.sensors.external_task import ExternalTaskMarker -from airflow.sensors.external_task import ExternalTaskSensor -from airflow.utils.task_group import TaskGroup -import datetime -from operators.gcp_container_operator import GKEPodOperator -from utils.constants import ALLOWED_STATES, FAILED_STATES -from utils.gcp import bigquery_etl_query, bigquery_dq_check - -docs = """ -### bqetl_mobile_kpi_metrics - -Built from bigquery-etl repo, [`dags/bqetl_mobile_kpi_metrics.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_mobile_kpi_metrics.py) - -#### Description - -Generates support metrics for mobile KPI's -#### Owner - -kik@mozilla.com - -#### Tags - -* impact/tier_1 -* repo/bigquery-etl -""" - - -default_args = { - "owner": "kik@mozilla.com", - "start_date": datetime.datetime(2024, 6, 3, 0, 0), - "end_date": None, - "email": ["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - "depends_on_past": False, - "retry_delay": datetime.timedelta(seconds=1800), - "email_on_failure": True, - "email_on_retry": False, - "retries": 1, -} - -tags = ["impact/tier_1", "repo/bigquery-etl"] - -with DAG( - "bqetl_mobile_kpi_metrics", - default_args=default_args, - schedule_interval="0 12 * * *", - doc_md=docs, - tags=tags, -) as dag: - - task_group_fenix = TaskGroup("fenix") - - task_group_firefox_ios = TaskGroup("firefox_ios") - - task_group_focus_android = TaskGroup("focus_android") - - task_group_focus_ios = TaskGroup("focus_ios") - - task_group_klar_android = TaskGroup("klar_android") - - task_group_klar_ios = TaskGroup("klar_ios") - - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 = ( - ExternalTaskSensor( - task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - external_dag_id="bqetl_analytics_tables", - external_task_id="checks__fail_fenix_derived__firefox_android_clients__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - ) - - wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fenix_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_firefox_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - external_dag_id="bqetl_firefox_ios", - external_task_id="checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - execution_delta=datetime.timedelta(seconds=28800), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_ios.checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_ios.org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_android.checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_android.org_mozilla_klar_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_ios.checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_ios.org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - fenix_derived__engagement__v1 = bigquery_etl_query( - task_id="fenix_derived__engagement__v1", - destination_table="engagement_v1", - dataset_id="fenix_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_fenix, - ) - - fenix_derived__retention__v1 = bigquery_etl_query( - task_id="fenix_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="fenix_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_fenix, - ) - - firefox_ios_derived__engagement__v1 = bigquery_etl_query( - task_id="firefox_ios_derived__engagement__v1", - destination_table="engagement_v1", - dataset_id="firefox_ios_derived", - project_id="moz-fx-data-shared-prod", - ```

⚠️ Only part of the diff is displayed.

Link to full diff

[dataops-ci-bot]

Integration report for "Merge branch 'main' into RS_1233_add_is_enterprise_policies_to_search"

sql.diff

Click to expand!

```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_search.py /tmp/workspace/generated-sql/dags/bqetl_search.py --- /tmp/workspace/main-generated-sql/dags/bqetl_search.py 2024-06-05 14:07:24.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_search.py 2024-06-05 14:08:58.000000000 +0000 @@ -52,6 +52,18 @@ tags=tags, ) as dag: + wait_for_copy_deduplicate_main_ping = ExternalTaskSensor( + task_id="wait_for_copy_deduplicate_main_ping", + external_dag_id="copy_deduplicate", + external_task_id="copy_deduplicate_main_ping", + execution_delta=datetime.timedelta(seconds=7200), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + wait_for_telemetry_derived__clients_daily_joined__v1 = ExternalTaskSensor( task_id="wait_for_telemetry_derived__clients_daily_joined__v1", external_dag_id="bqetl_main_summary", @@ -262,6 +274,10 @@ ) search_derived__search_clients_daily__v8.set_upstream( + wait_for_copy_deduplicate_main_ping + ) + + search_derived__search_clients_daily__v8.set_upstream( wait_for_telemetry_derived__clients_daily_joined__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/query.sql 2024-06-05 14:03:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/query.sql 2024-06-05 14:03:56.000000000 +0000 @@ -13,6 +13,7 @@ os, os_version, is_default_browser, + policies_is_enterprise, channel, CAST( NULL AS STRING @@ -47,4 +48,5 @@ os, os_version, is_default_browser, + policies_is_enterprise, channel diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/schema.yaml 2024-06-05 14:03:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_aggregates_v8/schema.yaml 2024-06-05 14:03:56.000000000 +0000 @@ -42,6 +42,9 @@ name: is_default_browser type: BOOLEAN - mode: NULLABLE + name: policies_is_enterprise + type: BOOLEAN +- mode: NULLABLE name: channel type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql 2024-06-05 14:03:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/query.sql 2024-06-05 14:03:56.000000000 +0000 @@ -51,6 +51,26 @@ client_id, submission_date ), +is_enterprise_policies AS ( + SELECT + client_id, + DATE(submission_timestamp) AS submission_date, + mozfun.stats.mode_last( + ARRAY_AGG( + payload.processes.parent.scalars.policies_is_enterprise + ORDER BY + submission_timestamp + ) + ) AS policies_is_enterprise + FROM + `moz-fx-data-shared-prod`.telemetry_stable.main_v5 + WHERE + normalized_app_name = 'Firefox' + AND document_id IS NOT NULL + GROUP BY + client_id, + submission_date +), combined_access_point AS ( SELECT * EXCEPT (has_adblocker_addon), @@ -105,6 +125,9 @@ LEFT JOIN clients_with_adblocker_addons USING (client_id, submission_date) + LEFT JOIN + is_enterprise_policies + USING (client_id, submission_date) ), augmented AS ( SELECT @@ -229,6 +252,7 @@ country, get_search_addon_version(active_addons) AS addon_version, has_adblocker_addon, + policies_is_enterprise, app_version, distribution_id, locale, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml 2024-06-05 14:03:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/search_clients_daily_v8/schema.yaml 2024-06-05 14:03:56.000000000 +0000 @@ -45,6 +45,9 @@ name: is_default_browser type: BOOLEAN - mode: NULLABLE + name: policies_is_enterprise + type: BOOLEAN +- mode: NULLABLE name: profile_creation_date type: INTEGER - mode: NULLABLE ```

Link to full diff

[dataops-ci-bot]

Integration report for "Fix the tests"

sql.diff

Click to expand!

```diff Only in /tmp/workspace/main-generated-sql/dags/: bqetl_desktop_conv_evnt_categorization.py Only in /tmp/workspace/main-generated-sql/dags/: bqetl_mobile_kpi_metrics.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_analytics_tables.py /tmp/workspace/generated-sql/dags/bqetl_analytics_tables.py --- /tmp/workspace/main-generated-sql/dags/bqetl_analytics_tables.py 2024-06-05 14:03:37.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_analytics_tables.py 2024-06-05 14:15:33.000000000 +0000 @@ -265,13 +265,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_feature_usage__wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", external_dag_id="bqetl_mobile_feature_usage", external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", @@ -324,13 +317,6 @@ ) ExternalTaskMarker( - task_id="bqetl_desktop_conv_evnt_categorization__wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", - external_dag_id="bqetl_desktop_conv_evnt_categorization", - external_task_id="wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mozilla_org_derived__wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", external_dag_id="bqetl_mozilla_org_derived", external_task_id="wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_braze.py /tmp/workspace/generated-sql/dags/bqetl_braze.py --- /tmp/workspace/main-generated-sql/dags/bqetl_braze.py 2024-06-05 14:03:37.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_braze.py 2024-06-05 14:15:34.000000000 +0000 @@ -149,7 +149,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__changed_newsletters_sync__v1 = bigquery_etl_query( @@ -162,7 +161,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__changed_products_sync__v1 = bigquery_etl_query( @@ -175,7 +173,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__changed_users__v1 = bigquery_etl_query( @@ -200,7 +197,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__changed_waitlists_sync__v1 = bigquery_etl_query( @@ -213,7 +209,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__delete_users_sync__v1 = bigquery_etl_query( @@ -226,7 +221,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__users_previous_day_snapshot__v1 = bigquery_etl_query( @@ -354,7 +348,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -368,7 +361,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -382,7 +374,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -409,7 +400,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -423,7 +413,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -437,7 +426,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_desktop_conv_evnt_categorization.py /tmp/workspace/generated-sql/dags/bqetl_desktop_conv_evnt_categorization.py --- /tmp/workspace/main-generated-sql/dags/bqetl_desktop_conv_evnt_categorization.py 2024-06-05 14:03:37.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_desktop_conv_evnt_categorization.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,116 +0,0 @@ -# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py - -from airflow import DAG -from airflow.sensors.external_task import ExternalTaskMarker -from airflow.sensors.external_task import ExternalTaskSensor -from airflow.utils.task_group import TaskGroup -import datetime -from operators.gcp_container_operator import GKEPodOperator -from utils.constants import ALLOWED_STATES, FAILED_STATES -from utils.gcp import bigquery_etl_query, bigquery_dq_check - -docs = """ -### bqetl_desktop_conv_evnt_categorization - -Built from bigquery-etl repo, [`dags/bqetl_desktop_conv_evnt_categorization.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_desktop_conv_evnt_categorization.py) - -#### Description - -Loads the desktop conversion event tables -#### Owner - -kwindau@mozilla.com - -#### Tags - -* impact/tier_2 -* repo/bigquery-etl -""" - - -default_args = { - "owner": "kwindau@mozilla.com", - "start_date": datetime.datetime(2024, 6, 4, 0, 0), - "end_date": None, - "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], - "depends_on_past": False, - "retry_delay": datetime.timedelta(seconds=1800), - "email_on_failure": True, - "email_on_retry": False, - "retries": 2, -} - -tags = ["impact/tier_2", "repo/bigquery-etl"] - -with DAG( - "bqetl_desktop_conv_evnt_categorization", - default_args=default_args, - schedule_interval="0 12 * * *", - doc_md=docs, - tags=tags, -) as dag: - - wait_for_checks__fail_telemetry_derived__clients_first_seen__v2 = ( - ExternalTaskSensor( - task_id="wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", - external_dag_id="bqetl_analytics_tables", - external_task_id="checks__fail_telemetry_derived__clients_first_seen__v2", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - ) - - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 = ExternalTaskSensor( - task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - external_dag_id="bqetl_main_summary", - external_task_id="checks__fail_telemetry_derived__clients_last_seen__v2", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - checks__warn_google_ads_derived__conversion_event_categorization__v1 = bigquery_dq_check( - task_id="checks__warn_google_ads_derived__conversion_event_categorization__v1", - source_table='conversion_event_categorization_v1${{ macros.ds_format(macros.ds_add(ds, -14), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="google_ads_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], - depends_on_past=False, - parameters=["report_date:DATE:{{macros.ds_add(ds, -14)}}"] - + ["submission_date:DATE:{{ds}}"], - retries=0, - ) - - google_ads_derived__conversion_event_categorization__v1 = bigquery_etl_query( - task_id="google_ads_derived__conversion_event_categorization__v1", - destination_table='conversion_event_categorization_v1${{ macros.ds_format(macros.ds_add(ds, -14), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="google_ads_derived", - project_id="moz-fx-data-shared-prod", - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["report_date:DATE:{{macros.ds_add(ds, -14)}}"] - + ["submission_date:DATE:{{ds}}"], - ) - - checks__warn_google_ads_derived__conversion_event_categorization__v1.set_upstream( - google_ads_derived__conversion_event_categorization__v1 - ) - - google_ads_derived__conversion_event_categorization__v1.set_upstream( - wait_for_checks__fail_telemetry_derived__clients_first_seen__v2 - ) - - google_ads_derived__conversion_event_categorization__v1.set_upstream( - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 - ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_desktop_retention_model.py /tmp/workspace/generated-sql/dags/bqetl_desktop_retention_model.py --- /tmp/workspace/main-generated-sql/dags/bqetl_desktop_retention_model.py 2024-06-05 14:03:37.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_desktop_retention_model.py 2024-06-05 14:15:34.000000000 +0000 @@ -90,15 +90,13 @@ telemetry_derived__desktop_retention__v1 = bigquery_etl_query( task_id="telemetry_derived__desktop_retention__v1", - destination_table='desktop_retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', + destination_table="desktop_retention_v1", dataset_id="telemetry_derived", project_id="moz-fx-data-shared-prod", owner="mhirose@mozilla.com", email=["mhirose@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, + date_partition_parameter="submission_date", depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], ) telemetry_derived__desktop_retention_clients__v1 = bigquery_etl_query( diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_firefox_ios.py /tmp/workspace/generated-sql/dags/bqetl_firefox_ios.py --- /tmp/workspace/main-generated-sql/dags/bqetl_firefox_ios.py 2024-06-05 14:03:37.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_firefox_ios.py 2024-06-05 14:15:31.000000000 +0000 @@ -311,13 +311,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=57600)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_feature_usage__wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", external_dag_id="bqetl_mobile_feature_usage", external_task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_glean_usage.py /tmp/workspace/generated-sql/dags/bqetl_glean_usage.py --- /tmp/workspace/main-generated-sql/dags/bqetl_glean_usage.py 2024-06-05 14:03:37.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_glean_usage.py 2024-06-05 14:15:37.000000000 +0000 @@ -365,6 +365,20 @@ task_group=task_group_burnham, ) + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( + task_id="checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1", + source_table="baseline_clients_last_seen_v1", + dataset_id="accounts_cirrus_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=True, + owner="ascholtz@mozilla.com", + email=["ascholtz@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_accounts_cirrus, + ) + checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -461,13 +475,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", @@ -524,13 +531,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", @@ -587,13 +587,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", @@ -650,13 +643,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", @@ -713,13 +699,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", @@ -764,21 +743,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1_external", - parent_group=task_group_focus_android, - ) as checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1_external.set_upstream( - checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 - ) - checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -798,13 +762,6 @@ parent_group=task_group_focus_android, ) as checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1_external: ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_nondesktop__wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_nondesktop", external_task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", @@ -836,21 +793,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1_external", - parent_group=task_group_focus_android, - ) as checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1_external.set_upstream( - checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 - ) - checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -877,13 +819,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_activation__wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_mobile_activation", external_task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", @@ -934,13 +869,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_activation__wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_mobile_activation", external_task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", @@ -977,13 +905,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_activation__wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_mobile_activation", external_task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", @@ -1013,13 +934,6 @@ parent_group=task_group_focus_ios, ) as checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1_external: ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_nondesktop__wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_nondesktop", external_task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", @@ -1056,13 +970,6 @@ parent_group=task_group_klar_ios, ) as checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1_external: ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_nondesktop__wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_nondesktop", external_task_id="wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", @@ -1135,13 +1042,6 @@ parent_group=task_group_klar_android, ) as checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1_external: ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_nondesktop__wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_nondesktop", external_task_id="wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", @@ -1259,6 +1159,20 @@ task_group=task_group_pine, ) + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( + task_id="checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1", + source_table="baseline_clients_last_seen_v1", + dataset_id="accounts_cirrus_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=False, + owner="ascholtz@mozilla.com", + email=["ascholtz@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_accounts_cirrus, + ) + checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -3192,13 +3106,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", @@ -3281,13 +3188,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", @@ -3372,13 +3272,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", @@ -3463,13 +3356,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", @@ -3556,13 +3442,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", @@ -3741,21 +3620,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "org_mozilla_focus_beta_derived__baseline_clients_daily__v1_external", - parent_group=task_group_focus_android, - ) as org_mozilla_focus_beta_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_focus_beta_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_focus_beta_derived__baseline_clients_daily__v1 - ) - org_mozilla_focus_beta_derived__baseline_clients_first_seen__v1 = ( bigquery_etl_query( task_id="org_mozilla_focus_beta_derived__baseline_clients_first_seen__v1", @@ -3813,21 +3677,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "org_mozilla_focus_derived__baseline_clients_daily__v1_external", - parent_group=task_group_focus_android, - ) as org_mozilla_focus_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_focus_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_focus_derived__baseline_clients_daily__v1 - ) - org_mozilla_focus_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_focus_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -3883,21 +3732,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "org_mozilla_focus_nightly_derived__baseline_clients_daily__v1_external", - parent_group=task_group_focus_android, - ) as org_mozilla_focus_nightly_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_focus_nightly_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_focus_nightly_derived__baseline_clients_daily__v1 - ) - org_mozilla_focus_nightly_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_focus_nightly_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -3973,13 +3807,6 @@ execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", ) - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - org_mozilla_ios_fennec_derived__baseline_clients_daily__v1_external.set_upstream( org_mozilla_ios_fennec_derived__baseline_clients_daily__v1 ) @@ -4059,13 +3886,6 @@ execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", ) - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - org_mozilla_ios_firefox_derived__baseline_clients_daily__v1_external.set_upstream( org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 ) @@ -4149,13 +3969,6 @@ execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", ) - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1_external.set_upstream( org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 ) @@ -4325,21 +4138,6 @@ task_group=task_group_focus_ios, ) - with TaskGroup( - "org_mozilla_ios_focus_derived__baseline_clients_daily__v1_external", - parent_group=task_group_focus_ios, - ) as org_mozilla_ios_focus_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_ios_focus_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_ios_focus_derived__baseline_clients_daily__v1 - ) - org_mozilla_ios_focus_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_ios_focus_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -4395,21 +4193,6 @@ task_group=task_group_klar_ios, ) - with TaskGroup( - "org_mozilla_ios_klar_derived__baseline_clients_daily__v1_external", - parent_group=task_group_klar_ios, - ) as org_mozilla_ios_klar_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_ios_klar_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_ios_klar_derived__baseline_clients_daily__v1 - ) - org_mozilla_ios_klar_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_ios_klar_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -4524,21 +4307,6 @@ task_group=task_group_klar_android, ) - with TaskGroup( - "org_mozilla_klar_derived__baseline_clients_daily__v1_external", - parent_group=task_group_klar_android, - ) as org_mozilla_klar_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_klar_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_klar_derived__baseline_clients_daily__v1 - ) - org_mozilla_klar_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_klar_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -5100,6 +4868,18 @@ burnham_derived__metrics_clients_daily__v1 ) + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + accounts_cirrus_derived__baseline_clients_last_seen__v1 + ) + + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 + ) + checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1.set_upstream( checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 ) @@ -5384,6 +5164,18 @@ pine_derived__baseline_clients_last_seen__v1 ) + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + accounts_cirrus_derived__baseline_clients_last_seen__v1 + ) + + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 + ) + checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1.set_upstream( checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_main_summary.py /tmp/workspace/generated-sql/dags/bqetl_main_summary.py --- /tmp/workspace/main-generated-sql/dags/bqetl_main_summary.py 2024-06-05 14:03:37.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_main_summary.py 2024-06-05 14:15:30.000000000 +0000 @@ -144,13 +144,6 @@ ) ExternalTaskMarker( - task_id="bqetl_desktop_conv_evnt_categorization__wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - external_dag_id="bqetl_desktop_conv_evnt_categorization", - external_task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_search_dashboard__wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", external_dag_id="bqetl_search_dashboard", external_task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_mobile_kpi_metrics.py /tmp/workspace/generated-sql/dags/bqetl_mobile_kpi_metrics.py --- /tmp/workspace/main-generated-sql/dags/bqetl_mobile_kpi_metrics.py 2024-06-05 14:03:37.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_mobile_kpi_metrics.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,765 +0,0 @@ -# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py - -from airflow import DAG -from airflow.sensors.external_task import ExternalTaskMarker -from airflow.sensors.external_task import ExternalTaskSensor -from airflow.utils.task_group import TaskGroup -import datetime -from operators.gcp_container_operator import GKEPodOperator -from utils.constants import ALLOWED_STATES, FAILED_STATES -from utils.gcp import bigquery_etl_query, bigquery_dq_check - -docs = """ -### bqetl_mobile_kpi_metrics - -Built from bigquery-etl repo, [`dags/bqetl_mobile_kpi_metrics.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_mobile_kpi_metrics.py) - -#### Description - -Generates support metrics for mobile KPI's -#### Owner - -kik@mozilla.com - -#### Tags - -* impact/tier_1 -* repo/bigquery-etl -""" - - -default_args = { - "owner": "kik@mozilla.com", - "start_date": datetime.datetime(2024, 6, 3, 0, 0), - "end_date": None, - "email": ["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - "depends_on_past": False, - "retry_delay": datetime.timedelta(seconds=1800), - "email_on_failure": True, - "email_on_retry": False, - "retries": 1, -} - -tags = ["impact/tier_1", "repo/bigquery-etl"] - -with DAG( - "bqetl_mobile_kpi_metrics", - default_args=default_args, - schedule_interval="0 12 * * *", - doc_md=docs, - tags=tags, -) as dag: - - task_group_fenix = TaskGroup("fenix") - - task_group_firefox_ios = TaskGroup("firefox_ios") - - task_group_focus_android = TaskGroup("focus_android") - - task_group_focus_ios = TaskGroup("focus_ios") - - task_group_klar_android = TaskGroup("klar_android") - - task_group_klar_ios = TaskGroup("klar_ios") - - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 = ( - ExternalTaskSensor( - task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - external_dag_id="bqetl_analytics_tables", - external_task_id="checks__fail_fenix_derived__firefox_android_clients__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - ) - - wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fenix_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_firefox_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - external_dag_id="bqetl_firefox_ios", - external_task_id="checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - execution_delta=datetime.timedelta(seconds=28800), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_ios.checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_ios.org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_android.checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_android.org_mozilla_klar_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_ios.checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_ios.org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - fenix_derived__engagement__v1 = bigquery_etl_query( - task_id="fenix_derived__engagement__v1", - destination_table="engagement_v1", - dataset_id="fenix_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_fenix, - ) - - fenix_derived__retention__v1 = bigquery_etl_query( - task_id="fenix_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="fenix_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_fenix, - ) - - firefox_ios_derived__engagement__v1 = bigquery_etl_query( - task_id="firefox_ios_derived__engagement__v1", - destination_table="engagement_v1", - dataset_id="firefox_ios_derived", - project_id="moz-fx-data-shared-prod", - ```

⚠️ Only part of the diff is displayed.

Link to full diff