dataops-ci-bot
commented
2 months ago Integration report for "feat: add attribution_clients template to mobile_kpi_support_metrics sql generator"
sql.diff
Click to expand!
```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_analytics_aggregations.py /tmp/workspace/generated-sql/dags/bqetl_analytics_aggregations.py --- /tmp/workspace/main-generated-sql/dags/bqetl_analytics_aggregations.py 2024-07-24 14:54:57.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_analytics_aggregations.py 2024-07-24 15:13:34.000000000 +0000 @@ -190,42 +190,6 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - wait_for_firefox_ios_derived__metrics_clients_last_seen__v1 = ExternalTaskSensor( task_id="wait_for_firefox_ios_derived__metrics_clients_last_seen__v1", external_dag_id="bqetl_glean_usage", @@ -238,42 +202,6 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - wait_for_focus_android_derived__metrics_clients_last_seen__v1 = ExternalTaskSensor( task_id="wait_for_focus_android_derived__metrics_clients_last_seen__v1", external_dag_id="bqetl_glean_usage", @@ -298,18 +226,6 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_ios.checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - wait_for_focus_ios_derived__metrics_clients_last_seen__v1 = ExternalTaskSensor( task_id="wait_for_focus_ios_derived__metrics_clients_last_seen__v1", external_dag_id="bqetl_glean_usage", @@ -322,18 +238,6 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_android.checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - wait_for_klar_android_derived__metrics_clients_last_seen__v1 = ExternalTaskSensor( task_id="wait_for_klar_android_derived__metrics_clients_last_seen__v1", external_dag_id="bqetl_glean_usage", @@ -346,18 +250,6 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_ios.checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - wait_for_klar_ios_derived__metrics_clients_last_seen__v1 = ExternalTaskSensor( task_id="wait_for_klar_ios_derived__metrics_clients_last_seen__v1", external_dag_id="bqetl_glean_usage", @@ -893,26 +785,6 @@ ) fenix_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 - ) - - fenix_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 - ) - - fenix_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 - ) - - fenix_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 - ) - - fenix_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 - ) - - fenix_active_users_aggregates.set_upstream( wait_for_fenix_derived__metrics_clients_last_seen__v1 ) @@ -929,34 +801,10 @@ ) firefox_ios_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 - ) - - firefox_ios_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 - ) - - firefox_ios_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 - ) - - firefox_ios_active_users_aggregates.set_upstream( wait_for_firefox_ios_derived__metrics_clients_last_seen__v1 ) focus_android_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 - ) - - focus_android_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 - ) - - focus_android_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 - ) - - focus_android_active_users_aggregates.set_upstream( wait_for_focus_android_derived__metrics_clients_last_seen__v1 ) @@ -965,26 +813,14 @@ ) focus_ios_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 - ) - - focus_ios_active_users_aggregates.set_upstream( wait_for_focus_ios_derived__metrics_clients_last_seen__v1 ) klar_android_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1 - ) - - klar_android_active_users_aggregates.set_upstream( wait_for_klar_android_derived__metrics_clients_last_seen__v1 ) klar_ios_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1 - ) - - klar_ios_active_users_aggregates.set_upstream( wait_for_klar_ios_derived__metrics_clients_last_seen__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_analytics_tables.py /tmp/workspace/generated-sql/dags/bqetl_analytics_tables.py --- /tmp/workspace/main-generated-sql/dags/bqetl_analytics_tables.py 2024-07-24 14:54:57.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_analytics_tables.py 2024-07-24 15:13:34.000000000 +0000 @@ -265,13 +265,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_feature_usage__wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", external_dag_id="bqetl_mobile_feature_usage", external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_firefox_ios.py /tmp/workspace/generated-sql/dags/bqetl_firefox_ios.py --- /tmp/workspace/main-generated-sql/dags/bqetl_firefox_ios.py 2024-07-24 14:54:57.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_firefox_ios.py 2024-07-24 15:13:33.000000000 +0000 @@ -299,13 +299,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=57600)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_feature_usage__wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", external_dag_id="bqetl_mobile_feature_usage", external_task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_generated_funnels.py /tmp/workspace/generated-sql/dags/bqetl_generated_funnels.py --- /tmp/workspace/main-generated-sql/dags/bqetl_generated_funnels.py 2024-07-24 14:54:57.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_generated_funnels.py 2024-07-24 15:13:35.000000000 +0000 @@ -276,6 +276,21 @@ ) ) + monitor_frontend_derived__monitor_dashboard_user_journey_funnels__v1 = bigquery_etl_query( + task_id="monitor_frontend_derived__monitor_dashboard_user_journey_funnels__v1", + destination_table="monitor_dashboard_user_journey_funnels_v1", + dataset_id="monitor_frontend_derived", + project_id="moz-fx-data-shared-prod", + owner="ksiegler@mozilla.org", + email=[ + "ascholtz@mozilla.com", + "ksiegler@mozilla.org", + "telemetry-alerts@mozilla.com", + ], + date_partition_parameter="submission_date", + depends_on_past=False, + ) + accounts_frontend_derived__email_first_reg_login_funnels__v1.set_upstream( wait_for_copy_deduplicate_all ) @@ -325,3 +340,7 @@ firefox_accounts_derived__registration_funnels_legacy_events__v1.set_upstream( wait_for_firefox_accounts_derived__fxa_stdout_events__v1 ) + + monitor_frontend_derived__monitor_dashboard_user_journey_funnels__v1.set_upstream( + wait_for_copy_deduplicate_all + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_mobile_kpi_metrics.py /tmp/workspace/generated-sql/dags/bqetl_mobile_kpi_metrics.py --- /tmp/workspace/main-generated-sql/dags/bqetl_mobile_kpi_metrics.py 2024-07-24 14:54:57.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_mobile_kpi_metrics.py 2024-07-24 15:13:36.000000000 +0000 @@ -62,277 +62,11 @@ task_group_klar_ios = TaskGroup("klar_ios") - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 = ( - ExternalTaskSensor( - task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - external_dag_id="bqetl_analytics_tables", - external_task_id="checks__fail_fenix_derived__firefox_android_clients__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - ) - - wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fenix_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_firefox_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - external_dag_id="bqetl_firefox_ios", - external_task_id="checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - execution_delta=datetime.timedelta(seconds=28800), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), + wait_for_copy_deduplicate_all = ExternalTaskSensor( + task_id="wait_for_copy_deduplicate_all", + external_dag_id="copy_deduplicate", + external_task_id="copy_deduplicate_all", + execution_delta=datetime.timedelta(seconds=39600), check_existence=True, mode="reschedule", allowed_states=ALLOWED_STATES, @@ -340,93 +74,9 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_ios.checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_ios.org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_android.checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_android.org_mozilla_klar_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_ios.checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_ios.org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - checks__warn_fenix_derived__engagement__v1 = bigquery_dq_check( - task_id="checks__warn_fenix_derived__engagement__v1", - source_table="engagement_v1", + checks__warn_fenix_derived__attribution_clients__v1 = bigquery_dq_check( + task_id="checks__warn_fenix_derived__attribution_clients__v1", + source_table="attribution_clients_v1", dataset_id="fenix_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=False, @@ -438,24 +88,9 @@ task_group=task_group_fenix, ) - checks__warn_fenix_derived__retention__v1 = bigquery_dq_check( - task_id="checks__warn_fenix_derived__retention__v1", - source_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="fenix_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - retries=0, - task_group=task_group_fenix, - ) - - checks__warn_firefox_ios_derived__engagement__v1 = bigquery_dq_check( - task_id="checks__warn_firefox_ios_derived__engagement__v1", - source_table="engagement_v1", + checks__warn_firefox_ios_derived__attribution_clients__v1 = bigquery_dq_check( + task_id="checks__warn_firefox_ios_derived__attribution_clients__v1", + source_table="attribution_clients_v1", dataset_id="firefox_ios_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=False, @@ -467,24 +102,9 @@ task_group=task_group_firefox_ios, ) - checks__warn_firefox_ios_derived__retention__v1 = bigquery_dq_check( - task_id="checks__warn_firefox_ios_derived__retention__v1", - source_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="firefox_ios_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - retries=0, - task_group=task_group_firefox_ios, - ) - - checks__warn_focus_android_derived__engagement__v1 = bigquery_dq_check( - task_id="checks__warn_focus_android_derived__engagement__v1", - source_table="engagement_v1", + checks__warn_focus_android_derived__attribution_clients__v1 = bigquery_dq_check( + task_id="checks__warn_focus_android_derived__attribution_clients__v1", + source_table="attribution_clients_v1", dataset_id="focus_android_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=False, @@ -496,24 +116,9 @@ task_group=task_group_focus_android, ) - checks__warn_focus_android_derived__retention__v1 = bigquery_dq_check( - task_id="checks__warn_focus_android_derived__retention__v1", - source_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="focus_android_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - retries=0, - task_group=task_group_focus_android, - ) - - checks__warn_focus_ios_derived__engagement__v1 = bigquery_dq_check( - task_id="checks__warn_focus_ios_derived__engagement__v1", - source_table="engagement_v1", + checks__warn_focus_ios_derived__attribution_clients__v1 = bigquery_dq_check( + task_id="checks__warn_focus_ios_derived__attribution_clients__v1", + source_table="attribution_clients_v1", dataset_id="focus_ios_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=False, @@ -525,24 +130,9 @@ task_group=task_group_focus_ios, ) - checks__warn_focus_ios_derived__retention__v1 = bigquery_dq_check( - task_id="checks__warn_focus_ios_derived__retention__v1", - source_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="focus_ios_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - retries=0, - task_group=task_group_focus_ios, - ) - - checks__warn_klar_android_derived__engagement__v1 = bigquery_dq_check( - task_id="checks__warn_klar_android_derived__engagement__v1", - source_table="engagement_v1", + checks__warn_klar_android_derived__attribution_clients__v1 = bigquery_dq_check( + task_id="checks__warn_klar_android_derived__attribution_clients__v1", + source_table="attribution_clients_v1", dataset_id="klar_android_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=False, @@ -554,24 +144,9 @@ task_group=task_group_klar_android, ) - checks__warn_klar_android_derived__retention__v1 = bigquery_dq_check( - task_id="checks__warn_klar_android_derived__retention__v1", - source_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="klar_android_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - retries=0, - task_group=task_group_klar_android, - ) - - checks__warn_klar_ios_derived__engagement__v1 = bigquery_dq_check( - task_id="checks__warn_klar_ios_derived__engagement__v1", - source_table="engagement_v1", + checks__warn_klar_ios_derived__attribution_clients__v1 = bigquery_dq_check( + task_id="checks__warn_klar_ios_derived__attribution_clients__v1", + source_table="attribution_clients_v1", dataset_id="klar_ios_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=False, @@ -583,24 +158,9 @@ task_group=task_group_klar_ios, ) - checks__warn_klar_ios_derived__retention__v1 = bigquery_dq_check( - task_id="checks__warn_klar_ios_derived__retention__v1", - source_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="klar_ios_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - retries=0, - task_group=task_group_klar_ios, - ) - - fenix_derived__engagement__v1 = bigquery_etl_query( - task_id="fenix_derived__engagement__v1", - destination_table="engagement_v1", + fenix_derived__attribution_clients__v1 = bigquery_etl_query( + task_id="fenix_derived__attribution_clients__v1", + destination_table="attribution_clients_v1", dataset_id="fenix_derived", project_id="moz-fx-data-shared-prod", owner="mozilla/kpi_table_reviewers", @@ -610,23 +170,9 @@ task_group=task_group_fenix, ) - fenix_derived__retention__v1 = bigquery_etl_query( - task_id="fenix_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="fenix_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_fenix, - ) - - firefox_ios_derived__engagement__v1 = bigquery_etl_query( - task_id="firefox_ios_derived__engagement__v1", - destination_table="engagement_v1", + firefox_ios_derived__attribution_clients__v1 = bigquery_etl_query( + task_id="firefox_ios_derived__attribution_clients__v1", + destination_table="attribution_clients_v1", dataset_id="firefox_ios_derived", project_id="moz-fx-data-shared-prod", owner="mozilla/kpi_table_reviewers", @@ -636,23 +182,9 @@ task_group=task_group_firefox_ios, ) - firefox_ios_derived__retention__v1 = bigquery_etl_query( - task_id="firefox_ios_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="firefox_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_firefox_ios, - ) - - focus_android_derived__engagement__v1 = bigquery_etl_query( - task_id="focus_android_derived__engagement__v1", - destination_table="engagement_v1", + focus_android_derived__attribution_clients__v1 = bigquery_etl_query( + task_id="focus_android_derived__attribution_clients__v1", + destination_table="attribution_clients_v1", dataset_id="focus_android_derived", project_id="moz-fx-data-shared-prod", owner="mozilla/kpi_table_reviewers", @@ -662,23 +194,9 @@ task_group=task_group_focus_android, ) - focus_android_derived__retention__v1 = bigquery_etl_query( - task_id="focus_android_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="focus_android_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_focus_android, - ) - - focus_ios_derived__engagement__v1 = bigquery_etl_query( - task_id="focus_ios_derived__engagement__v1", - destination_table="engagement_v1", + focus_ios_derived__attribution_clients__v1 = bigquery_etl_query( + task_id="focus_ios_derived__attribution_clients__v1", + destination_table="attribution_clients_v1", dataset_id="focus_ios_derived", project_id="moz-fx-data-shared-prod", owner="mozilla/kpi_table_reviewers", @@ -688,23 +206,9 @@ task_group=task_group_focus_ios, ) - focus_ios_derived__retention__v1 = bigquery_etl_query( - task_id="focus_ios_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="focus_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_focus_ios, - ) - - klar_android_derived__engagement__v1 = bigquery_etl_query( - task_id="klar_android_derived__engagement__v1", - destination_table="engagement_v1", + klar_android_derived__attribution_clients__v1 = bigquery_etl_query( + task_id="klar_android_derived__attribution_clients__v1", + destination_table="attribution_clients_v1", dataset_id="klar_android_derived", project_id="moz-fx-data-shared-prod", owner="mozilla/kpi_table_reviewers", @@ -714,23 +218,9 @@ task_group=task_group_klar_android, ) - klar_android_derived__retention__v1 = bigquery_etl_query( - task_id="klar_android_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="klar_android_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_klar_android, - ) - - klar_ios_derived__engagement__v1 = bigquery_etl_query( - task_id="klar_ios_derived__engagement__v1", - destination_table="engagement_v1", + klar_ios_derived__attribution_clients__v1 = bigquery_etl_query( + task_id="klar_ios_derived__attribution_clients__v1", + destination_table="attribution_clients_v1", dataset_id="klar_ios_derived", project_id="moz-fx-data-shared-prod", owner="mozilla/kpi_table_reviewers", @@ -740,246 +230,32 @@ task_group=task_group_klar_ios, ) - klar_ios_derived__retention__v1 = bigquery_etl_query( - task_id="klar_ios_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="klar_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_klar_ios, - ) - - checks__warn_fenix_derived__engagement__v1.set_upstream( - fenix_derived__engagement__v1 - ) - - checks__warn_fenix_derived__retention__v1.set_upstream(fenix_derived__retention__v1) - - checks__warn_firefox_ios_derived__engagement__v1.set_upstream( - firefox_ios_derived__engagement__v1 - ) - - checks__warn_firefox_ios_derived__retention__v1.set_upstream( - firefox_ios_derived__retention__v1 - ) - - checks__warn_focus_android_derived__engagement__v1.set_upstream( - focus_android_derived__engagement__v1 - ) - - checks__warn_focus_android_derived__retention__v1.set_upstream( - focus_android_derived__retention__v1 - ) - - checks__warn_focus_ios_derived__engagement__v1.set_upstream( - focus_ios_derived__engagement__v1 - ) - - checks__warn_focus_ios_derived__retention__v1.set_upstream( - focus_ios_derived__retention__v1 - ) - - checks__warn_klar_android_derived__engagement__v1.set_upstream( - klar_android_derived__engagement__v1 - ) - - checks__warn_klar_android_derived__retention__v1.set_upstream( - klar_android_derived__retention__v1 - ) - - checks__warn_klar_ios_derived__engagement__v1.set_upstream( - klar_ios_derived__engagement__v1 - ) - - checks__warn_klar_ios_derived__retention__v1.set_upstream( - klar_ios_derived__retention__v1 - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1 + checks__warn_fenix_derived__attribution_clients__v1.set_upstream( + fenix_derived__attribution_clients__v1 ) - fenix_derived__retention__v1.set_upstream( - wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1 + checks__warn_firefox_ios_derived__attribution_clients__v1.set_upstream( + firefox_ios_derived__attribution_clients__v1 ) - fenix_derived__retention__v1.set_upstream( - wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1 + checks__warn_focus_android_derived__attribution_clients__v1.set_upstream( + focus_android_derived__attribution_clients__v1 ) - fenix_derived__retention__v1.set_upstream( - wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1 + checks__warn_focus_ios_derived__attribution_clients__v1.set_upstream( + focus_ios_derived__attribution_clients__v1 ) - firefox_ios_derived__engagement__v1.set_upstream( - wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 + checks__warn_klar_android_derived__attribution_clients__v1.set_upstream( + klar_android_derived__attribution_clients__v1 ) - firefox_ios_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 + checks__warn_klar_ios_derived__attribution_clients__v1.set_upstream( + klar_ios_derived__attribution_clients__v1 ) - firefox_ios_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 - ) - - firefox_ios_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 - ) - - firefox_ios_derived__retention__v1.set_upstream( - wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 - ) - - firefox_ios_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 - ) - - firefox_ios_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 - ) - - firefox_ios_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 - ) - - firefox_ios_derived__retention__v1.set_upstream( - wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1 - ) - - firefox_ios_derived__retention__v1.set_upstream( - wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 - ) - - firefox_ios_derived__retention__v1.set_upstream( - wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 - ) - - focus_android_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 - ) - - focus_android_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 - ) - - focus_android_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 - ) - - focus_android_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 - ) - - focus_android_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 - ) - - focus_android_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 - ) - - focus_android_derived__retention__v1.set_upstream( - wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1 - ) - - focus_android_derived__retention__v1.set_upstream( - wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1 - ) - - focus_android_derived__retention__v1.set_upstream( - wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1 - ) - - focus_ios_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 - ) - - focus_ios_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 - ) - - focus_ios_derived__retention__v1.set_upstream( - wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1 - ) - - klar_android_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1 - ) - - klar_android_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1 - ) - - klar_android_derived__retention__v1.set_upstream( - wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1 - ) - - klar_ios_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1 - ) - - klar_ios_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1 - ) + fenix_derived__attribution_clients__v1.set_upstream(wait_for_copy_deduplicate_all) - klar_ios_derived__retention__v1.set_upstream( - wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1 + firefox_ios_derived__attribution_clients__v1.set_upstream( + wait_for_copy_deduplicate_all ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_search_dashboard.py /tmp/workspace/generated-sql/dags/bqetl_search_dashboard.py --- /tmp/workspace/main-generated-sql/dags/bqetl_search_dashboard.py 2024-07-24 14:54:57.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_search_dashboard.py 2024-07-24 15:13:33.000000000 +0000 @@ -83,174 +83,6 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_ios.checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existe ```⚠️ Only part of the diff is displayed.
bani
kik-kik
feat(): add attribution_clients template to mobile_kpi_support_metrics sql generator
Checklist for reviewer:
<username>:<branch>of the fork as parameter. The parameter will also show up in the logs of themanual-trigger-required-for-forkCI task together with more detailed instructions.For modifications to schemas in restricted namespaces (see
CODEOWNERS):┆Issue is synchronized with this Jira Task