dataops-ci-bot
commented
4 months ago Integration report for "feature(glam): Add non-normalized histograms on Glean metrics"
sql.diff
Click to expand!
```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_mobile_engagement_model.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_acoustic_contact_export.py /tmp/workspace/generated-sql/dags/bqetl_acoustic_contact_export.py --- /tmp/workspace/main-generated-sql/dags/bqetl_acoustic_contact_export.py 2024-05-30 17:09:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_acoustic_contact_export.py 2024-05-30 17:20:28.000000000 +0000 @@ -99,6 +99,20 @@ depends_on_past=False, ) + with TaskGroup( + "acoustic_external__contact_raw__v1_external", + ) as acoustic_external__contact_raw__v1_external: + ExternalTaskMarker( + task_id="bqetl_braze__wait_for_acoustic_external__contact_raw__v1", + external_dag_id="bqetl_braze", + external_task_id="wait_for_acoustic_external__contact_raw__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=82800)).isoformat() }}", + ) + + acoustic_external__contact_raw__v1_external.set_upstream( + acoustic_external__contact_raw__v1 + ) + acoustic_derived__contact__v1.set_upstream(acoustic_external__contact_raw__v1) acoustic_derived__contact_current_snapshot__v1.set_upstream( diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_acoustic_raw_recipient_export.py /tmp/workspace/generated-sql/dags/bqetl_acoustic_raw_recipient_export.py --- /tmp/workspace/main-generated-sql/dags/bqetl_acoustic_raw_recipient_export.py 2024-05-30 17:09:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_acoustic_raw_recipient_export.py 2024-05-30 17:20:28.000000000 +0000 @@ -87,6 +87,20 @@ depends_on_past=False, ) + with TaskGroup( + "acoustic_external__raw_recipient_raw__v1_external", + ) as acoustic_external__raw_recipient_raw__v1_external: + ExternalTaskMarker( + task_id="bqetl_braze__wait_for_acoustic_external__raw_recipient_raw__v1", + external_dag_id="bqetl_braze", + external_task_id="wait_for_acoustic_external__raw_recipient_raw__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=82800)).isoformat() }}", + ) + + acoustic_external__raw_recipient_raw__v1_external.set_upstream( + acoustic_external__raw_recipient_raw__v1 + ) + acoustic_derived__raw_recipient__v1.set_upstream( acoustic_external__raw_recipient_raw__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_analytics_aggregations.py /tmp/workspace/generated-sql/dags/bqetl_analytics_aggregations.py --- /tmp/workspace/main-generated-sql/dags/bqetl_analytics_aggregations.py 2024-05-30 17:09:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_analytics_aggregations.py 2024-05-30 17:20:29.000000000 +0000 @@ -140,10 +140,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 = ExternalTaskSensor( - task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", + wait_for_telemetry_derived__clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_telemetry_derived__clients_last_seen__v1", external_dag_id="bqetl_main_summary", - external_task_id="checks__fail_telemetry_derived__clients_last_seen__v2", + external_task_id="telemetry_derived__clients_last_seen__v1", execution_delta=datetime.timedelta(seconds=8100), check_existence=True, mode="reschedule", @@ -288,18 +288,6 @@ ) ) - wait_for_telemetry_derived__clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_telemetry_derived__clients_last_seen__v1", - external_dag_id="bqetl_main_summary", - external_task_id="telemetry_derived__clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - active_users_aggregates_attribution_v1 = bigquery_etl_query( task_id="active_users_aggregates_attribution_v1", destination_table="active_users_aggregates_attribution_v1", @@ -330,6 +318,20 @@ depends_on_past=False, ) + with TaskGroup( + "active_users_aggregates_device_v1_external", + ) as active_users_aggregates_device_v1_external: + ExternalTaskMarker( + task_id="bqetl_search_dashboard__wait_for_active_users_aggregates_device_v1", + external_dag_id="bqetl_search_dashboard", + external_task_id="wait_for_active_users_aggregates_device_v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", + ) + + active_users_aggregates_device_v1_external.set_upstream( + active_users_aggregates_device_v1 + ) + active_users_aggregates_v1 = bigquery_etl_query( task_id="active_users_aggregates_v1", destination_table="active_users_aggregates_v1", @@ -362,20 +364,6 @@ retries=0, ) - with TaskGroup( - "checks__fail_fenix_derived__active_users_aggregates__v3_external", - ) as checks__fail_fenix_derived__active_users_aggregates__v3_external: - ExternalTaskMarker( - task_id="bqetl_search_dashboard__wait_for_checks__fail_fenix_derived__active_users_aggregates__v3", - external_dag_id="bqetl_search_dashboard", - external_task_id="wait_for_checks__fail_fenix_derived__active_users_aggregates__v3", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", - ) - - checks__fail_fenix_derived__active_users_aggregates__v3_external.set_upstream( - checks__fail_fenix_derived__active_users_aggregates__v3 - ) - checks__fail_firefox_ios_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__fail_firefox_ios_derived__active_users_aggregates__v3", source_table="active_users_aggregates_v3", @@ -393,20 +381,6 @@ retries=0, ) - with TaskGroup( - "checks__fail_firefox_ios_derived__active_users_aggregates__v3_external", - ) as checks__fail_firefox_ios_derived__active_users_aggregates__v3_external: - ExternalTaskMarker( - task_id="bqetl_search_dashboard__wait_for_checks__fail_firefox_ios_derived__active_users_aggregates__v3", - external_dag_id="bqetl_search_dashboard", - external_task_id="wait_for_checks__fail_firefox_ios_derived__active_users_aggregates__v3", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", - ) - - checks__fail_firefox_ios_derived__active_users_aggregates__v3_external.set_upstream( - checks__fail_firefox_ios_derived__active_users_aggregates__v3 - ) - checks__fail_focus_android_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__fail_focus_android_derived__active_users_aggregates__v3", source_table="active_users_aggregates_v3", @@ -424,20 +398,6 @@ retries=0, ) - with TaskGroup( - "checks__fail_focus_android_derived__active_users_aggregates__v3_external", - ) as checks__fail_focus_android_derived__active_users_aggregates__v3_external: - ExternalTaskMarker( - task_id="bqetl_search_dashboard__wait_for_checks__fail_focus_android_derived__active_users_aggregates__v3", - external_dag_id="bqetl_search_dashboard", - external_task_id="wait_for_checks__fail_focus_android_derived__active_users_aggregates__v3", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", - ) - - checks__fail_focus_android_derived__active_users_aggregates__v3_external.set_upstream( - checks__fail_focus_android_derived__active_users_aggregates__v3 - ) - checks__fail_focus_ios_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__fail_focus_ios_derived__active_users_aggregates__v3", source_table="active_users_aggregates_v3", @@ -455,20 +415,6 @@ retries=0, ) - with TaskGroup( - "checks__fail_focus_ios_derived__active_users_aggregates__v3_external", - ) as checks__fail_focus_ios_derived__active_users_aggregates__v3_external: - ExternalTaskMarker( - task_id="bqetl_search_dashboard__wait_for_checks__fail_focus_ios_derived__active_users_aggregates__v3", - external_dag_id="bqetl_search_dashboard", - external_task_id="wait_for_checks__fail_focus_ios_derived__active_users_aggregates__v3", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", - ) - - checks__fail_focus_ios_derived__active_users_aggregates__v3_external.set_upstream( - checks__fail_focus_ios_derived__active_users_aggregates__v3 - ) - checks__fail_klar_ios_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__fail_klar_ios_derived__active_users_aggregates__v3", source_table="active_users_aggregates_v3", @@ -486,20 +432,6 @@ retries=0, ) - with TaskGroup( - "checks__fail_klar_ios_derived__active_users_aggregates__v3_external", - ) as checks__fail_klar_ios_derived__active_users_aggregates__v3_external: - ExternalTaskMarker( - task_id="bqetl_search_dashboard__wait_for_checks__fail_klar_ios_derived__active_users_aggregates__v3", - external_dag_id="bqetl_search_dashboard", - external_task_id="wait_for_checks__fail_klar_ios_derived__active_users_aggregates__v3", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", - ) - - checks__fail_klar_ios_derived__active_users_aggregates__v3_external.set_upstream( - checks__fail_klar_ios_derived__active_users_aggregates__v3 - ) - checks__warn_fenix_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__warn_fenix_derived__active_users_aggregates__v3", source_table="active_users_aggregates_v3", @@ -634,20 +566,6 @@ depends_on_past=False, ) - with TaskGroup( - "firefox_desktop_active_users_aggregates_external", - ) as firefox_desktop_active_users_aggregates_external: - ExternalTaskMarker( - task_id="bqetl_search_dashboard__wait_for_firefox_desktop_active_users_aggregates", - external_dag_id="bqetl_search_dashboard", - external_task_id="wait_for_firefox_desktop_active_users_aggregates", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", - ) - - firefox_desktop_active_users_aggregates_external.set_upstream( - firefox_desktop_active_users_aggregates - ) - firefox_ios_active_users_aggregates = bigquery_etl_query( task_id="firefox_ios_active_users_aggregates", destination_table="active_users_aggregates_v3", @@ -821,11 +739,11 @@ ) checks__warn_firefox_desktop_derived__active_users_aggregates__v3.set_upstream( - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 + firefox_desktop_active_users_aggregates ) checks__warn_firefox_desktop_derived__active_users_aggregates__v3.set_upstream( - firefox_desktop_active_users_aggregates + wait_for_telemetry_derived__clients_last_seen__v1 ) checks__warn_firefox_ios_derived__active_users_aggregates__v3.set_upstream( @@ -865,7 +783,7 @@ ) firefox_desktop_active_users_aggregates.set_upstream( - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 + wait_for_telemetry_derived__clients_last_seen__v1 ) firefox_ios_active_users_aggregates.set_upstream( diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_analytics_tables.py /tmp/workspace/generated-sql/dags/bqetl_analytics_tables.py --- /tmp/workspace/main-generated-sql/dags/bqetl_analytics_tables.py 2024-05-30 17:09:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_analytics_tables.py 2024-05-30 17:20:29.000000000 +0000 @@ -211,10 +211,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 = ExternalTaskSensor( - task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", + wait_for_telemetry_derived__clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_telemetry_derived__clients_last_seen__v1", external_dag_id="bqetl_main_summary", - external_task_id="checks__fail_telemetry_derived__clients_last_seen__v2", + external_task_id="telemetry_derived__clients_last_seen__v1", check_existence=True, mode="reschedule", allowed_states=ALLOWED_STATES, @@ -284,6 +284,13 @@ external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", ) + ExternalTaskMarker( + task_id="bqetl_mobile_engagement_model__wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", + external_dag_id="bqetl_mobile_engagement_model", + external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", + ) + checks__fail_fenix_derived__firefox_android_clients__v1_external.set_upstream( checks__fail_fenix_derived__firefox_android_clients__v1 ) @@ -679,5 +686,5 @@ ) telemetry_derived__clients_first_seen_28_days_later__v1.set_upstream( - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 + wait_for_telemetry_derived__clients_last_seen__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_braze.py /tmp/workspace/generated-sql/dags/bqetl_braze.py --- /tmp/workspace/main-generated-sql/dags/bqetl_braze.py 2024-05-30 17:09:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_braze.py 2024-05-30 17:20:30.000000000 +0000 @@ -50,7 +50,7 @@ with DAG( "bqetl_braze", default_args=default_args, - schedule_interval="30 4,12,20 * * *", + schedule_interval="0 10 * * *", doc_md=docs, tags=tags, ) as dag: @@ -59,6 +59,7 @@ task_id="wait_for_subscription_platform_derived__logical_subscriptions_history__v1", external_dag_id="bqetl_subplat", external_task_id="subscription_platform_derived__logical_subscriptions_history__v1", + execution_delta=datetime.timedelta(seconds=29700), check_existence=True, mode="reschedule", allowed_states=ALLOWED_STATES, @@ -70,6 +71,31 @@ task_id="wait_for_checks__fail_marketing_suppression_list_derived__main_suppression_list__v1", external_dag_id="bqetl_marketing_suppression_list", external_task_id="checks__fail_marketing_suppression_list_derived__main_suppression_list__v1", + execution_delta=datetime.timedelta(seconds=25200), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_acoustic_external__contact_raw__v1 = ExternalTaskSensor( + task_id="wait_for_acoustic_external__contact_raw__v1", + external_dag_id="bqetl_acoustic_contact_export", + external_task_id="acoustic_external__contact_raw__v1", + execution_delta=datetime.timedelta(seconds=3600), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_acoustic_external__raw_recipient_raw__v1 = ExternalTaskSensor( + task_id="wait_for_acoustic_external__raw_recipient_raw__v1", + external_dag_id="bqetl_acoustic_raw_recipient_export", + external_task_id="acoustic_external__raw_recipient_raw__v1", + execution_delta=datetime.timedelta(seconds=3600), check_existence=True, mode="reschedule", allowed_states=ALLOWED_STATES, @@ -125,6 +151,18 @@ sql_file_path="sql/moz-fx-data-shared-prod/braze_derived/subscriptions_map_v1/script.sql", ) + braze_derived__suppressions__v1 = bigquery_etl_query( + task_id="braze_derived__suppressions__v1", + destination_table="suppressions_v1", + dataset_id="braze_derived", + project_id="moz-fx-data-shared-prod", + owner="cbeck@mozilla.com", + email=["cbeck@mozilla.com", "leli@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + task_concurrency=1, + ) + braze_derived__user_profiles__v1 = bigquery_etl_query( task_id="braze_derived__user_profiles__v1", destination_table="user_profiles_v1", @@ -491,6 +529,8 @@ braze_derived__newsletters__v1.set_upstream(checks__fail_braze_derived__users__v1) + braze_derived__products__v1.set_upstream(checks__fail_braze_derived__users__v1) + braze_derived__products__v1.set_upstream( wait_for_subscription_platform_derived__logical_subscriptions_history__v1 ) @@ -503,6 +543,10 @@ checks__fail_braze_derived__user_profiles__v1 ) + braze_derived__suppressions__v1.set_upstream( + wait_for_checks__fail_marketing_suppression_list_derived__main_suppression_list__v1 + ) + braze_derived__user_profiles__v1.set_upstream( checks__fail_braze_derived__newsletters__v1 ) @@ -517,6 +561,16 @@ checks__fail_braze_derived__waitlists__v1 ) + braze_derived__users__v1.set_upstream(wait_for_acoustic_external__contact_raw__v1) + + braze_derived__users__v1.set_upstream( + wait_for_acoustic_external__raw_recipient_raw__v1 + ) + + braze_derived__users__v1.set_upstream( + checks__fail_braze_derived__subscriptions_map__v1 + ) + braze_derived__users__v1.set_upstream( wait_for_checks__fail_marketing_suppression_list_derived__main_suppression_list__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_desktop_funnel.py /tmp/workspace/generated-sql/dags/bqetl_desktop_funnel.py --- /tmp/workspace/main-generated-sql/dags/bqetl_desktop_funnel.py 2024-05-30 17:09:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_desktop_funnel.py 2024-05-30 17:20:27.000000000 +0000 @@ -52,11 +52,11 @@ tags=tags, ) as dag: - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 = ExternalTaskSensor( - task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - external_dag_id="bqetl_main_summary", - external_task_id="checks__fail_telemetry_derived__clients_last_seen__v2", - execution_delta=datetime.timedelta(seconds=7200), + wait_for_copy_deduplicate_all = ExternalTaskSensor( + task_id="wait_for_copy_deduplicate_all", + external_dag_id="copy_deduplicate", + external_task_id="copy_deduplicate_all", + execution_delta=datetime.timedelta(seconds=10800), check_existence=True, mode="reschedule", allowed_states=ALLOWED_STATES, @@ -64,11 +64,11 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_copy_deduplicate_all = ExternalTaskSensor( - task_id="wait_for_copy_deduplicate_all", - external_dag_id="copy_deduplicate", - external_task_id="copy_deduplicate_all", - execution_delta=datetime.timedelta(seconds=10800), + wait_for_telemetry_derived__clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_telemetry_derived__clients_last_seen__v1", + external_dag_id="bqetl_main_summary", + external_task_id="telemetry_derived__clients_last_seen__v1", + execution_delta=datetime.timedelta(seconds=7200), check_existence=True, mode="reschedule", allowed_states=ALLOWED_STATES, @@ -110,11 +110,11 @@ ) telemetry_derived__desktop_funnel_activation_day_6__v1.set_upstream( - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 + wait_for_copy_deduplicate_all ) telemetry_derived__desktop_funnel_activation_day_6__v1.set_upstream( - wait_for_copy_deduplicate_all + wait_for_telemetry_derived__clients_last_seen__v1 ) telemetry_derived__desktop_funnel_installs__v1.set_upstream( diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_feature_usage.py /tmp/workspace/generated-sql/dags/bqetl_feature_usage.py --- /tmp/workspace/main-generated-sql/dags/bqetl_feature_usage.py 2024-05-30 17:09:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_feature_usage.py 2024-05-30 17:20:28.000000000 +0000 @@ -70,18 +70,6 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 = ExternalTaskSensor( - task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - external_dag_id="bqetl_main_summary", - external_task_id="checks__fail_telemetry_derived__clients_last_seen__v2", - execution_delta=datetime.timedelta(seconds=10800), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - wait_for_copy_deduplicate_all = ExternalTaskSensor( task_id="wait_for_copy_deduplicate_all", external_dag_id="copy_deduplicate", @@ -118,6 +106,18 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) + wait_for_telemetry_derived__clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_telemetry_derived__clients_last_seen__v1", + external_dag_id="bqetl_main_summary", + external_task_id="telemetry_derived__clients_last_seen__v1", + execution_delta=datetime.timedelta(seconds=10800), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + wait_for_telemetry_derived__main_remainder_1pct__v1 = ExternalTaskSensor( task_id="wait_for_telemetry_derived__main_remainder_1pct__v1", external_dag_id="bqetl_main_summary", @@ -149,10 +149,6 @@ telemetry_derived__feature_usage__v2.set_upstream(wait_for_bq_main_events) - telemetry_derived__feature_usage__v2.set_upstream( - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 - ) - telemetry_derived__feature_usage__v2.set_upstream(wait_for_copy_deduplicate_all) telemetry_derived__feature_usage__v2.set_upstream(wait_for_event_events) @@ -162,5 +158,9 @@ ) telemetry_derived__feature_usage__v2.set_upstream( + wait_for_telemetry_derived__clients_last_seen__v1 + ) + + telemetry_derived__feature_usage__v2.set_upstream( wait_for_telemetry_derived__main_remainder_1pct__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_firefox_ios.py /tmp/workspace/generated-sql/dags/bqetl_firefox_ios.py --- /tmp/workspace/main-generated-sql/dags/bqetl_firefox_ios.py 2024-05-30 17:09:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_firefox_ios.py 2024-05-30 17:20:28.000000000 +0000 @@ -287,6 +287,13 @@ execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", ) + ExternalTaskMarker( + task_id="bqetl_mobile_engagement_model__wait_for_checks__fail_firefox_ios_derived__clients_activation__v1", + external_dag_id="bqetl_mobile_engagement_model", + external_task_id="wait_for_checks__fail_firefox_ios_derived__clients_activation__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=57600)).isoformat() }}", + ) + checks__fail_firefox_ios_derived__clients_activation__v1_external.set_upstream( checks__fail_firefox_ios_derived__clients_activation__v1 ) @@ -336,6 +343,13 @@ execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", ) + ExternalTaskMarker( + task_id="bqetl_mobile_engagement_model__wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", + external_dag_id="bqetl_mobile_engagement_model", + external_task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=57600)).isoformat() }}", + ) + checks__fail_firefox_ios_derived__firefox_ios_clients__v1_external.set_upstream( checks__fail_firefox_ios_derived__firefox_ios_clients__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_glean_usage.py /tmp/workspace/generated-sql/dags/bqetl_glean_usage.py --- /tmp/workspace/main-generated-sql/dags/bqetl_glean_usage.py 2024-05-30 17:09:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_glean_usage.py 2024-05-30 17:20:32.000000000 +0000 @@ -495,6 +495,13 @@ ) ExternalTaskMarker( + task_id="bqetl_mobile_engagement_model__wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_mobile_engagement_model", + external_task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", + ) + + ExternalTaskMarker( task_id="bqetl_gud__wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_gud", external_task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", @@ -551,6 +558,13 @@ ) ExternalTaskMarker( + task_id="bqetl_mobile_engagement_model__wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_mobile_engagement_model", + external_task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", + ) + + ExternalTaskMarker( task_id="bqetl_gud__wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_gud", external_task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", @@ -607,6 +621,13 @@ ) ExternalTaskMarker( + task_id="bqetl_mobile_engagement_model__wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_mobile_engagement_model", + external_task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", + ) + + ExternalTaskMarker( task_id="bqetl_gud__wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_gud", external_task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", @@ -663,6 +684,13 @@ ) ExternalTaskMarker( + task_id="bqetl_mobile_engagement_model__wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_mobile_engagement_model", + external_task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", + ) + + ExternalTaskMarker( task_id="bqetl_gud__wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_gud", external_task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", @@ -719,6 +747,13 @@ ) ExternalTaskMarker( + task_id="bqetl_mobile_engagement_model__wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_mobile_engagement_model", + external_task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", + ) + + ExternalTaskMarker( task_id="bqetl_gud__wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_gud", external_task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", @@ -743,6 +778,21 @@ task_group=task_group_focus_android, ) + with TaskGroup( + "checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1_external", + parent_group=task_group_focus_android, + ) as checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1_external: + ExternalTaskMarker( + task_id="bqetl_mobile_engagement_model__wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_mobile_engagement_model", + external_task_id="wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", + ) + + checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1_external.set_upstream( + checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 + ) + checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -769,6 +819,13 @@ ) ExternalTaskMarker( + task_id="bqetl_mobile_engagement_model__wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_mobile_engagement_model", + external_task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", + ) + + ExternalTaskMarker( task_id="bqetl_gud__wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_gud", external_task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", @@ -793,6 +850,21 @@ task_group=task_group_focus_android, ) + with TaskGroup( + "checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1_external", + parent_group=task_group_focus_android, + ) as checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1_external: + ExternalTaskMarker( + task_id="bqetl_mobile_engagement_model__wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_mobile_engagement_model", + external_task_id="wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", + ) + + checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1_external.set_upstream( + checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 + ) + checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -833,6 +905,13 @@ ) ExternalTaskMarker( + task_id="bqetl_mobile_engagement_model__wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_mobile_engagement_model", + external_task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", + ) + + ExternalTaskMarker( task_id="bqetl_gud__wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_gud", external_task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", @@ -875,6 +954,13 @@ execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", ) + ExternalTaskMarker( + task_id="bqetl_mobile_engagement_model__wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_mobile_engagement_model", + external_task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", + ) + checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1_external.set_upstream( checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 ) @@ -911,6 +997,13 @@ execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", ) + ExternalTaskMarker( + task_id="bqetl_mobile_engagement_model__wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_mobile_engagement_model", + external_task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", + ) + checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1_external.set_upstream( checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 ) @@ -941,6 +1034,13 @@ ) ExternalTaskMarker( + task_id="bqetl_mobile_engagement_model__wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_mobile_engagement_model", + external_task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", + ) + + ExternalTaskMarker( task_id="bqetl_gud__wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_gud", external_task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_gud.py /tmp/workspace/generated-sql/dags/bqetl_gud.py --- /tmp/workspace/main-generated-sql/dags/bqetl_gud.py 2024-05-30 17:09:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_gud.py 2024-05-30 17:20:29.000000000 +0000 @@ -50,10 +50,10 @@ tags=tags, ) as dag: - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 = ExternalTaskSensor( - task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", + wait_for_telemetry_derived__clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_telemetry_derived__clients_last_seen__v1", external_dag_id="bqetl_main_summary", - external_task_id="checks__fail_telemetry_derived__clients_last_seen__v2", + external_task_id="telemetry_derived__clients_last_seen__v1", execution_delta=datetime.timedelta(seconds=3600), check_existence=True, mode="reschedule", @@ -355,7 +355,7 @@ ) telemetry_derived__smoot_usage_desktop__v2.set_upstream( - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 + wait_for_telemetry_derived__clients_last_seen__v1 ) telemetry_derived__smoot_usage_desktop_compressed__v2.set_upstream( diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_kpis_shredder.py /tmp/workspace/generated-sql/dags/bqetl_kpis_shredder.py --- /tmp/workspace/main-generated-sql/dags/bqetl_kpis_shredder.py 2024-05-30 17:09:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_kpis_shredder.py 2024-05-30 17:20:29.000000000 +0000 @@ -86,10 +86,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 = ExternalTaskSensor( - task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", + wait_for_telemetry_derived__clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_telemetry_derived__clients_last_seen__v1", external_dag_id="bqetl_main_summary", - external_task_id="checks__fail_telemetry_derived__clients_last_seen__v2", + external_task_id="telemetry_derived__clients_last_seen__v1", check_existence=True, mode="reschedule", allowed_states=ALLOWED_STATES, @@ -218,15 +218,15 @@ ) firefox_desktop_active_users_aggregates_for_deletion_requests.set_upstream( - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 + wait_for_copy_deduplicate_all ) firefox_desktop_active_users_aggregates_for_deletion_requests.set_upstream( - wait_for_copy_deduplicate_all + wait_for_search_derived__mobile_search_clients_daily__v1 ) firefox_desktop_active_users_aggregates_for_deletion_requests.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 + wait_for_telemetry_derived__clients_last_seen__v1 ) firefox_ios_active_users_aggregates_for_deletion_requests.set_upstream( diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_main_summary.py /tmp/workspace/generated-sql/dags/bqetl_main_summary.py --- /tmp/workspace/main-generated-sql/dags/bqetl_main_summary.py 2024-05-30 17:09:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_main_summary.py 2024-05-30 17:20:27.000000000 +0000 @@ -131,32 +131,6 @@ "checks__fail_telemetry_derived__clients_last_seen__v2_external", ) as checks__fail_telemetry_derived__clients_last_seen__v2_external: ExternalTaskMarker( - task_id="bqetl_analytics_aggregations__wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - external_dag_id="bqetl_analytics_aggregations", - external_task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=78300)).isoformat() }}", - ) - - ExternalTaskMarker( - task_id="bqetl_kpis_shredder__wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - external_dag_id="bqetl_kpis_shredder", - external_task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - ) - - ExternalTaskMarker( - task_id="bqetl_search_dashboard__wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - external_dag_id="bqetl_search_dashboard", - external_task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=77400)).isoformat() }}", - ) - - ExternalTaskMarker( - task_id="bqetl_analytics_tables__wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - external_dag_id="bqetl_analytics_tables", - external_task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - ) - - ExternalTaskMarker( task_id="bqetl_desktop_engagement_model__wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", external_dag_id="bqetl_desktop_engagement_model", external_task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", @@ -164,13 +138,6 @@ ) ExternalTaskMarker( - task_id="bqetl_desktop_funnel__wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - external_dag_id="bqetl_desktop_funnel", - external_task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=79200)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_desktop_retention_model__wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", external_dag_id="bqetl_desktop_retention_model", external_task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", @@ -178,27 +145,6 @@ ) ExternalTaskMarker( - task_id="bqetl_feature_usage__wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - external_dag_id="bqetl_feature_usage", - external_task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=75600)).isoformat() }}", - ) - - ExternalTaskMarker( - task_id="bqetl_gud__wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - external_dag_id="bqetl_gud", - external_task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=82800)).isoformat() }}", - ) - - ExternalTaskMarker( - task_id="bqetl_unified__wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - external_dag_id="bqetl_unified", - external_task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=82800)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="taar_daily__wait_for_clients_last_seen", external_dag_id="taar_daily", external_task_id="wait_for_clients_last_seen", @@ -508,6 +454,26 @@ "telemetry_derived__clients_last_seen__v1_external", ) as telemetry_derived__clients_last_seen__v1_external: ExternalTaskMarker( + task_id="bqetl_analytics_aggregations__wait_for_telemetry_derived__clients_last_seen__v1", + external_dag_id="bqetl_analytics_aggregations", + external_task_id="wait_for_telemetry_derived__clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=78300)).isoformat() }}", + ) + + ExternalTaskMarker( + task_id="bqetl_kpis_shredder__wait_for_telemetry_derived__clients_last_seen__v1", + external_dag_id="bqetl_kpis_shredder", + external_task_id="wait_for_telemetry_derived__clients_last_seen__v1", + ) + + ExternalTaskMarker( + task_id="bqetl_search_dashboard__wait_for_telemetry_derived__clients_last_seen__v1", + external_dag_id="bqetl_search_dashboard", + external_task_id="wait_for_telemetry_derived__clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=77400)).isoformat() }}", + ) + + ExternalTaskMarker( task_id="bqetl_addons__wait_for_telemetry_derived__clients_last_seen__v1", external_dag_id="bqetl_addons", external_task_id="wait_for_telemetry_derived__clients_last_seen__v1", @@ -515,10 +481,37 @@ ) ExternalTaskMarker( - task_id="bqetl_analytics_aggregations__wait_for_telemetry_derived__clients_last_seen__v1", - external_dag_id="bqetl_analytics_aggregations", + task_id="bqetl_analytics_tables__wait_for_telemetry_derived__clients_last_seen__v1", + external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_telemetry_derived__clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=78300)).isoformat() }}", + ) + + ExternalTaskMarker( + task_id="bqetl_desktop_funnel__wait_for_telemetry_derived__clients_last_seen__v1", + external_dag_id="bqetl_desktop_funnel", + external_task_id="wait_for_telemetry_derived__clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=79200)).isoformat() }}", + ) + + ExternalTaskMarker( + task_id="bqetl_feature_usage__wait_for_telemetry_derived__clients_last_seen__v1", + external_dag_id="bqetl_feature_usage", + external_task_id="wait_for_telemetry_derived__clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=75600)).isoformat() }}", + ) + + ExternalTaskMarker( + task_id="bqetl_gud__wait_for_telemetry_derived__clients_last_seen__v1", + external_dag_id="bqetl_gud", + external_task_id="wait_for_telemetry_derived__clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=82800)).isoformat() }}", + ) + + ExternalTaskMarker( + task_id="bqetl_unified__wait_for_telemetry_derived__clients_last_seen__v1", + external_dag_id="bqetl_unified", + external_task_id="wait_for_telemetry_derived__clients_last_seen__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=82800)).isoformat() }}", ) ExternalTaskMarker( @@ -748,7 +741,7 @@ crashes_daily_v1.set_upstream(wait_for_copy_deduplicate_all) firefox_desktop_exact_mau28_by_client_count_dimensions.set_upstream( - checks__fail_telemetry_derived__clients_last_seen__v2 + telemetry_derived__clients_last_seen__v1 ) firefox_desktop_exact_mau28_by_dimensions.set_upstream( @@ -756,7 +749,7 @@ ) firefox_desktop_exact_mau28_by_dimensions_v2.set_upstream( - checks__fail_telemetry_derived__clients_last_seen__v2 + telemetry_derived__clients_last_seen__v1 ) telemetry_derived__clients_daily__v6.set_upstream( @@ -767,10 +760,6 @@ telemetry_derived__clients_daily_event__v1.set_upstream(wait_for_event_events) - telemetry_derived__clients_daily_joined__v1.set_upstream( - checks__fail_telemetry_derived__clients_last_seen__v2 - ) - telemetry_derived__clients_daily_joined__v1.set_upstream(crashes_daily_v1) telemetry_derived__clients_daily_joined__v1.set_upstream( @@ -781,6 +770,10 @@ telemetry_derived__clients_daily_event__v1 ) + telemetry_derived__clients_daily_joined__v1.set_upstream( + telemetry_derived__clients_last_seen__v1 + ) + telemetry_derived__clients_first_seen__v1.set_upstream( telemetry_derived__clients_daily__v6 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_marketing_suppression_list.py /tmp/workspace/generated-sql/dags/bqetl_marketing_suppression_list.py --- /tmp/workspace/main-generated-sql/dags/bqetl_marketing_suppression_list.py 2024-05-30 17:09:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_marketing_suppression_list.py 2024-05-30 17:20:30.000000000 +0000 @@ -125,6 +125,7 @@ task_id="bqetl_braze__wait_for_checks__fail_marketing_suppression_list_derived__main_suppression_list__v1", external_dag_id="bqetl_braze", external_task_id="wait_for_checks__fail_marketing_suppression_list_derived__main_suppression_list__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=61200)).isoformat() }}", ) checks__fail_marketing_suppression_list_derived__main_suppression_list__v1_external.set_upstream( diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_mobile_engagement_model.py /tmp/workspace/generated-sql/dags/bqetl_mobile_engagement_model.py --- /tmp/workspace/main-generated-sql/dags/bqetl_mobile_engagement_model.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_mobile_engagement_model.py 2024-05-30 17:20:30.000000000 +0000 @@ -0,0 +1,321 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_mobile_engagement_model + +Built from bigquery-etl repo, [`dags/bqetl_mobile_engagement_model.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_mobile_engagement_model.py) + +#### Description + +Loads the mobile engagement model tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 5, 16, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_mobile_engagement_model", + default_args=default_args, + schedule_interval="0 12 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", + external_dag_id="bqetl_analytics_tables", + external_task_id="checks__fail_fenix_derived__firefox_android_clients__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + wait_for_checks__fail_firefox_ios_derived__clients_activation__v1 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_firefox_ios_derived__clients_activation__v1", + external_dag_id="bqetl_firefox_ios", + external_task_id="checks__fail_firefox_ios_derived__clients_activation__v1", + execution_delta=datetime.timedelta(seconds=28800), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 = ExternalTaskSensor( + task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", + external_dag_id="bqetl_firefox_ios", + external_task_id="checks__fail_firefox_ios_derived__firefox_ios_clients__v1", + execution_delta=datetime.timedelta(seconds=28800), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="fenix.checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="fenix.checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="fenix.checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="fenix.checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="fenix.checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="focus_android.checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="focus_android.checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="focus_android.checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="firefox_ios.checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="focus_ios.checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + telemetry_derived__mobile_engagement__v1 = bigquery_etl_query( + task_id="telemetry_derived__mobile_engagement__v1", + destination_table="mobile_engagement_v1", + dataset_id="telemetry_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + ) + + telemetry_derived__mobile_engagement_clients__v1 = bigquery_etl_query( + task_id="telemetry_derived__mobile_engagement_clients__v1", + destination_table="mobile_engagement_clients_v1", + dataset_id="telemetry_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + ) + + telemetry_derived__mobile_engagement__v1.set_upstream( + telemetry_derived__mobile_engagement_clients__v1 + ) + + telemetry_derived__mobile_engagement_clients__v1.set_upstream( + wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 + ) + + telemetry_derived__mobile_engagement_clients__v1.set_upstream( + wait_for_checks__fail_firefox_ios_derived__clients_activation__v1 + ) + + telemetry_derived__mobile_engagement_clients__v1.set_upstream( + wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 + ) + + telemetry_derived__mobile_engagement_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 + ) + + telemetry_derived__mobile_engagement_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 + ) + + telemetry_derived__mobile_engagement_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 + ) + + telemetry_derived__mobile_engagement_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 + ) + + telemetry_derived__mobile_engagement_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 + ) + + telemetry_derived__mobile_engagement_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 + ) + + telemetry_derived__mobile_engagement_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 + ) + + telemetry_derived__mobile_engagement_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 + ) + + telemetry_derived__mobile_engagement_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 + ) + + telemetry_derived__mobile_engagement_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + telemetry_derived__mobile_engagement_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 + ) + + telemetry_derived__mobile_engagement_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_search_dashboard.py /tmp/workspace/generated-sql/dags/bqetl_search_dashboard.py --- /tmp/workspace/main-generated-sql/dags/bqetl_search_dashboard.py 2024-05-30 17:09:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_search_ ```⚠️ Only part of the diff is displayed.
Adds non-normalized aggregations to Glean metrics.
By default, GLAM aggregates histograms by normalizing on
client_id, which means clients have the same weight in their contribution to the histogram's values, regardless of how many times they submitted pings containing histograms. A non-normalized aggregation is a way to aggregate histograms in which submissions (instead of clients) have the same weight. This means that clients that send more submissions will have more weight on the final aggregation.Non-normalized aggregations were added to legacy telemetry probes in https://github.com/mozilla/bigquery-etl/pull/3873 and now I'm adding them to Glean metrics. While the first implementation duplicates the
bucket_countstask, this one adds the non-normalized data to the existing tasks, which is likely to be more efficient because it will scan less data. If that proves true I'll back-port it to legacy telemetry.Non-normalized data for Glean metrics will show up on GLAM starting from the day this gets merged. In other words, there will be no data backfill.
Checklist for reviewer:
<username>:<branch>of the fork as parameter. The parameter will also show up in the logs of themanual-trigger-required-for-forkCI task together with more detailed instructions.For modifications to schemas in restricted namespaces (see
CODEOWNERS):┆Issue is synchronized with this Jira Task