dataops-ci-bot
commented
1 month ago Integration report for "change status from Intitiate to Complete"
sql.diff
Click to expand!
```diff Only in /tmp/workspace/main-generated-sql/dags/: bqetl_mobile_kpi_metrics.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_analytics_tables.py /tmp/workspace/generated-sql/dags/bqetl_analytics_tables.py --- /tmp/workspace/main-generated-sql/dags/bqetl_analytics_tables.py 2024-06-04 17:01:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_analytics_tables.py 2024-06-04 17:13:13.000000000 +0000 @@ -265,13 +265,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_feature_usage__wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", external_dag_id="bqetl_mobile_feature_usage", external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_firefox_ios.py /tmp/workspace/generated-sql/dags/bqetl_firefox_ios.py --- /tmp/workspace/main-generated-sql/dags/bqetl_firefox_ios.py 2024-06-04 17:01:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_firefox_ios.py 2024-06-04 17:13:11.000000000 +0000 @@ -311,13 +311,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=57600)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_feature_usage__wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", external_dag_id="bqetl_mobile_feature_usage", external_task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_glean_usage.py /tmp/workspace/generated-sql/dags/bqetl_glean_usage.py --- /tmp/workspace/main-generated-sql/dags/bqetl_glean_usage.py 2024-06-04 17:01:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_glean_usage.py 2024-06-04 17:13:15.000000000 +0000 @@ -365,6 +365,20 @@ task_group=task_group_burnham, ) + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( + task_id="checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1", + source_table="baseline_clients_last_seen_v1", + dataset_id="accounts_cirrus_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=True, + owner="ascholtz@mozilla.com", + email=["ascholtz@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_accounts_cirrus, + ) + checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -461,13 +475,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", @@ -524,13 +531,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", @@ -587,13 +587,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", @@ -650,13 +643,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", @@ -713,13 +699,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", @@ -764,21 +743,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1_external", - parent_group=task_group_focus_android, - ) as checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1_external.set_upstream( - checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 - ) - checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -798,13 +762,6 @@ parent_group=task_group_focus_android, ) as checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1_external: ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_nondesktop__wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_nondesktop", external_task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", @@ -836,21 +793,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1_external", - parent_group=task_group_focus_android, - ) as checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1_external.set_upstream( - checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 - ) - checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -877,13 +819,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_activation__wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_mobile_activation", external_task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", @@ -934,13 +869,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_activation__wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_mobile_activation", external_task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", @@ -977,13 +905,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_activation__wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_mobile_activation", external_task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", @@ -1013,13 +934,6 @@ parent_group=task_group_focus_ios, ) as checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1_external: ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_nondesktop__wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_nondesktop", external_task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", @@ -1056,13 +970,6 @@ parent_group=task_group_klar_ios, ) as checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1_external: ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_nondesktop__wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_nondesktop", external_task_id="wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", @@ -1135,13 +1042,6 @@ parent_group=task_group_klar_android, ) as checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1_external: ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_nondesktop__wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_nondesktop", external_task_id="wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", @@ -1259,6 +1159,20 @@ task_group=task_group_pine, ) + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( + task_id="checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1", + source_table="baseline_clients_last_seen_v1", + dataset_id="accounts_cirrus_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=False, + owner="ascholtz@mozilla.com", + email=["ascholtz@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_accounts_cirrus, + ) + checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -3192,13 +3106,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", @@ -3281,13 +3188,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", @@ -3372,13 +3272,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", @@ -3463,13 +3356,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", @@ -3556,13 +3442,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_analytics_tables__wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", external_dag_id="bqetl_analytics_tables", external_task_id="wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", @@ -3741,21 +3620,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "org_mozilla_focus_beta_derived__baseline_clients_daily__v1_external", - parent_group=task_group_focus_android, - ) as org_mozilla_focus_beta_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_focus_beta_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_focus_beta_derived__baseline_clients_daily__v1 - ) - org_mozilla_focus_beta_derived__baseline_clients_first_seen__v1 = ( bigquery_etl_query( task_id="org_mozilla_focus_beta_derived__baseline_clients_first_seen__v1", @@ -3813,21 +3677,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "org_mozilla_focus_derived__baseline_clients_daily__v1_external", - parent_group=task_group_focus_android, - ) as org_mozilla_focus_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_focus_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_focus_derived__baseline_clients_daily__v1 - ) - org_mozilla_focus_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_focus_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -3883,21 +3732,6 @@ task_group=task_group_focus_android, ) - with TaskGroup( - "org_mozilla_focus_nightly_derived__baseline_clients_daily__v1_external", - parent_group=task_group_focus_android, - ) as org_mozilla_focus_nightly_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_focus_nightly_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_focus_nightly_derived__baseline_clients_daily__v1 - ) - org_mozilla_focus_nightly_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_focus_nightly_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -3973,13 +3807,6 @@ execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", ) - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - org_mozilla_ios_fennec_derived__baseline_clients_daily__v1_external.set_upstream( org_mozilla_ios_fennec_derived__baseline_clients_daily__v1 ) @@ -4059,13 +3886,6 @@ execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", ) - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - org_mozilla_ios_firefox_derived__baseline_clients_daily__v1_external.set_upstream( org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 ) @@ -4149,13 +3969,6 @@ execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", ) - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1_external.set_upstream( org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 ) @@ -4325,21 +4138,6 @@ task_group=task_group_focus_ios, ) - with TaskGroup( - "org_mozilla_ios_focus_derived__baseline_clients_daily__v1_external", - parent_group=task_group_focus_ios, - ) as org_mozilla_ios_focus_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_ios_focus_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_ios_focus_derived__baseline_clients_daily__v1 - ) - org_mozilla_ios_focus_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_ios_focus_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -4395,21 +4193,6 @@ task_group=task_group_klar_ios, ) - with TaskGroup( - "org_mozilla_ios_klar_derived__baseline_clients_daily__v1_external", - parent_group=task_group_klar_ios, - ) as org_mozilla_ios_klar_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_ios_klar_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_ios_klar_derived__baseline_clients_daily__v1 - ) - org_mozilla_ios_klar_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_ios_klar_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -4524,21 +4307,6 @@ task_group=task_group_klar_android, ) - with TaskGroup( - "org_mozilla_klar_derived__baseline_clients_daily__v1_external", - parent_group=task_group_klar_android, - ) as org_mozilla_klar_derived__baseline_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - org_mozilla_klar_derived__baseline_clients_daily__v1_external.set_upstream( - org_mozilla_klar_derived__baseline_clients_daily__v1 - ) - org_mozilla_klar_derived__baseline_clients_first_seen__v1 = bigquery_etl_query( task_id="org_mozilla_klar_derived__baseline_clients_first_seen__v1", destination_table="baseline_clients_first_seen_v1", @@ -5100,6 +4868,18 @@ burnham_derived__metrics_clients_daily__v1 ) + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + accounts_cirrus_derived__baseline_clients_last_seen__v1 + ) + + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 + ) + checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1.set_upstream( checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 ) @@ -5384,6 +5164,18 @@ pine_derived__baseline_clients_last_seen__v1 ) + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + accounts_cirrus_derived__baseline_clients_last_seen__v1 + ) + + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 + ) + checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1.set_upstream( checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_mobile_kpi_metrics.py /tmp/workspace/generated-sql/dags/bqetl_mobile_kpi_metrics.py --- /tmp/workspace/main-generated-sql/dags/bqetl_mobile_kpi_metrics.py 2024-06-04 17:01:10.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_mobile_kpi_metrics.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,765 +0,0 @@ -# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py - -from airflow import DAG -from airflow.sensors.external_task import ExternalTaskMarker -from airflow.sensors.external_task import ExternalTaskSensor -from airflow.utils.task_group import TaskGroup -import datetime -from operators.gcp_container_operator import GKEPodOperator -from utils.constants import ALLOWED_STATES, FAILED_STATES -from utils.gcp import bigquery_etl_query, bigquery_dq_check - -docs = """ -### bqetl_mobile_kpi_metrics - -Built from bigquery-etl repo, [`dags/bqetl_mobile_kpi_metrics.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_mobile_kpi_metrics.py) - -#### Description - -Generates support metrics for mobile KPI's -#### Owner - -kik@mozilla.com - -#### Tags - -* impact/tier_1 -* repo/bigquery-etl -""" - - -default_args = { - "owner": "kik@mozilla.com", - "start_date": datetime.datetime(2024, 6, 8, 0, 0), - "end_date": None, - "email": ["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - "depends_on_past": False, - "retry_delay": datetime.timedelta(seconds=1800), - "email_on_failure": True, - "email_on_retry": False, - "retries": 1, -} - -tags = ["impact/tier_1", "repo/bigquery-etl"] - -with DAG( - "bqetl_mobile_kpi_metrics", - default_args=default_args, - schedule_interval="0 12 * * *", - doc_md=docs, - tags=tags, -) as dag: - - task_group_fenix = TaskGroup("fenix") - - task_group_firefox_ios = TaskGroup("firefox_ios") - - task_group_focus_android = TaskGroup("focus_android") - - task_group_focus_ios = TaskGroup("focus_ios") - - task_group_klar_android = TaskGroup("klar_android") - - task_group_klar_ios = TaskGroup("klar_ios") - - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 = ( - ExternalTaskSensor( - task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - external_dag_id="bqetl_analytics_tables", - external_task_id="checks__fail_fenix_derived__firefox_android_clients__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - ) - - wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fenix_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_firefox_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - external_dag_id="bqetl_firefox_ios", - external_task_id="checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - execution_delta=datetime.timedelta(seconds=28800), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_beta_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_ios.checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_ios.org_mozilla_ios_focus_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_android.checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_android.org_mozilla_klar_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_ios.checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_ios.org_mozilla_ios_klar_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - fenix_derived__engagement__v1 = bigquery_etl_query( - task_id="fenix_derived__engagement__v1", - destination_table="engagement_v1", - dataset_id="fenix_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_fenix, - ) - - fenix_derived__retention__v1 = bigquery_etl_query( - task_id="fenix_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="fenix_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_fenix, - ) - - firefox_ios_derived__engagement__v1 = bigquery_etl_query( - task_id="firefox_ios_derived__engagement__v1", - destination_table="engagement_v1", - dataset_id="firefox_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_firefox_ios, - ) - - firefox_ios_derived__retention__v1 = bigquery_etl_query( - task_id="firefox_ios_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="firefox_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_firefox_ios, - ) - - focus_android_derived__engagement__v1 = bigquery_etl_query( - task_id="focus_android_derived__engagement__v1", - destination_table="engagement_v1", - dataset_id="focus_android_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_focus_android, - ) - - focus_android_derived__retention__v1 = bigquery_etl_query( - task_id="focus_android_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="focus_android_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_focus_android, - ) - - focus_ios_derived__engagement__v1 = bigquery_etl_query( - task_id="focus_ios_derived__engagement__v1", - destination_table="engagement_v1", - dataset_id="focus_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_focus_ios, - ) - - focus_ios_derived__retention__v1 = bigquery_etl_query( - task_id="focus_ios_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="focus_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_focus_ios, - ) - - klar_android_derived__engagement__v1 = bigquery_etl_query( - task_id="klar_android_derived__engagement__v1", - destination_table="engagement_v1", - dataset_id="klar_android_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_klar_android, - ) - - klar_android_derived__retention__v1 = bigquery_etl_query( - task_id="klar_android_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="klar_android_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_klar_android, - ) - - klar_ios_derived__engagement__v1 = bigquery_etl_query( - task_id="klar_ios_derived__engagement__v1", - destination_table="engagement_v1", - dataset_id="klar_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_klar_ios, - ) - - klar_ios_derived__retention__v1 = bigquery_etl_query( - task_id="klar_ios_derived__retention__v1", - destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="klar_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["metric_date:DATE:{{macros.ds_add(ds, -27)}}"] - + ["submission_date:DATE:{{ds}}"], - task_group=task_group_klar_ios, - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1 - ) - - fenix_derived__retention__v1.set_upstream( - wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1 - ) - - firefox_ios_derived__engagement__v1.set_upstream( - wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 - ) - - firefox_ios_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 - ) - - firefox_ios_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 - ) - - firefox_ios_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 - ) - - firefox_ios_derived__retention__v1.set_upstream( - wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 - ) - - firefox_ios_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 - ) - - firefox_ios_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 - ) - - firefox_ios_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 - ) - - firefox_ios_derived__retention__v1.set_upstream( - wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1 - ) - - firefox_ios_derived__retention__v1.set_upstream( - wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 - ) - - firefox_ios_derived__retention__v1.set_upstream( - wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 - ) - - focus_android_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 - ) - - focus_android_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 - ) - - focus_android_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 - ) - - focus_android_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 - ) - - focus_android_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 - ) - - focus_android_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 - ) - - focus_android_derived__retention__v1.set_upstream( - wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1 - ) - - focus_android_derived__retention__v1.set_upstream( - wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1 - ) - - focus_android_derived__retention__v1.set_upstream( - wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1 - ) - - focus_ios_derived__engagement__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 - ) - - focus_ios_derived__retention__v1.set_upstream( - wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 - ) - - focus_ios_derived__retention ```⚠️ Only part of the diff is displayed.
Finishing backfill of desktop_retention_v1.
Checklist for reviewer:
<username>:<branch>of the fork as parameter. The parameter will also show up in the logs of themanual-trigger-required-for-forkCI task together with more detailed instructions.For modifications to schemas in restricted namespaces (see
CODEOWNERS):┆Issue is synchronized with this Jira Task