dataops-ci-bot
commented
1 week ago Integration report for "feat: making further changes to introduce new_profiles and attribution models"
sql.diff
Click to expand!
```diff Only in /tmp/workspace/main-generated-sql/dags/: bqetl_firefox_desktop_ad_click_history.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_analytics_aggregations.py /tmp/workspace/generated-sql/dags/bqetl_analytics_aggregations.py --- /tmp/workspace/main-generated-sql/dags/bqetl_analytics_aggregations.py 2024-07-18 14:55:04.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_analytics_aggregations.py 2024-07-18 15:11:50.000000000 +0000 @@ -440,6 +440,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_fenix_derived__active_users_aggregates__v3_external", + ) as checks__fail_fenix_derived__active_users_aggregates__v3_external: + ExternalTaskMarker( + task_id="bqetl_search_dashboard__wait_for_checks__fail_fenix_derived__active_users_aggregates__v3", + external_dag_id="bqetl_search_dashboard", + external_task_id="wait_for_checks__fail_fenix_derived__active_users_aggregates__v3", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", + ) + + checks__fail_fenix_derived__active_users_aggregates__v3_external.set_upstream( + checks__fail_fenix_derived__active_users_aggregates__v3 + ) + checks__fail_firefox_ios_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__fail_firefox_ios_derived__active_users_aggregates__v3", source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', @@ -457,6 +471,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_firefox_ios_derived__active_users_aggregates__v3_external", + ) as checks__fail_firefox_ios_derived__active_users_aggregates__v3_external: + ExternalTaskMarker( + task_id="bqetl_search_dashboard__wait_for_checks__fail_firefox_ios_derived__active_users_aggregates__v3", + external_dag_id="bqetl_search_dashboard", + external_task_id="wait_for_checks__fail_firefox_ios_derived__active_users_aggregates__v3", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", + ) + + checks__fail_firefox_ios_derived__active_users_aggregates__v3_external.set_upstream( + checks__fail_firefox_ios_derived__active_users_aggregates__v3 + ) + checks__fail_focus_android_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__fail_focus_android_derived__active_users_aggregates__v3", source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', @@ -474,6 +502,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_focus_android_derived__active_users_aggregates__v3_external", + ) as checks__fail_focus_android_derived__active_users_aggregates__v3_external: + ExternalTaskMarker( + task_id="bqetl_search_dashboard__wait_for_checks__fail_focus_android_derived__active_users_aggregates__v3", + external_dag_id="bqetl_search_dashboard", + external_task_id="wait_for_checks__fail_focus_android_derived__active_users_aggregates__v3", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", + ) + + checks__fail_focus_android_derived__active_users_aggregates__v3_external.set_upstream( + checks__fail_focus_android_derived__active_users_aggregates__v3 + ) + checks__fail_focus_ios_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__fail_focus_ios_derived__active_users_aggregates__v3", source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', @@ -491,6 +533,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_focus_ios_derived__active_users_aggregates__v3_external", + ) as checks__fail_focus_ios_derived__active_users_aggregates__v3_external: + ExternalTaskMarker( + task_id="bqetl_search_dashboard__wait_for_checks__fail_focus_ios_derived__active_users_aggregates__v3", + external_dag_id="bqetl_search_dashboard", + external_task_id="wait_for_checks__fail_focus_ios_derived__active_users_aggregates__v3", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", + ) + + checks__fail_focus_ios_derived__active_users_aggregates__v3_external.set_upstream( + checks__fail_focus_ios_derived__active_users_aggregates__v3 + ) + checks__fail_klar_android_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__fail_klar_android_derived__active_users_aggregates__v3", source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', @@ -508,6 +564,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_klar_android_derived__active_users_aggregates__v3_external", + ) as checks__fail_klar_android_derived__active_users_aggregates__v3_external: + ExternalTaskMarker( + task_id="bqetl_search_dashboard__wait_for_checks__fail_klar_android_derived__active_users_aggregates__v3", + external_dag_id="bqetl_search_dashboard", + external_task_id="wait_for_checks__fail_klar_android_derived__active_users_aggregates__v3", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", + ) + + checks__fail_klar_android_derived__active_users_aggregates__v3_external.set_upstream( + checks__fail_klar_android_derived__active_users_aggregates__v3 + ) + checks__fail_klar_ios_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__fail_klar_ios_derived__active_users_aggregates__v3", source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', @@ -525,6 +595,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_klar_ios_derived__active_users_aggregates__v3_external", + ) as checks__fail_klar_ios_derived__active_users_aggregates__v3_external: + ExternalTaskMarker( + task_id="bqetl_search_dashboard__wait_for_checks__fail_klar_ios_derived__active_users_aggregates__v3", + external_dag_id="bqetl_search_dashboard", + external_task_id="wait_for_checks__fail_klar_ios_derived__active_users_aggregates__v3", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", + ) + + checks__fail_klar_ios_derived__active_users_aggregates__v3_external.set_upstream( + checks__fail_klar_ios_derived__active_users_aggregates__v3 + ) + checks__warn_fenix_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__warn_fenix_derived__active_users_aggregates__v3", source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', @@ -676,6 +760,20 @@ parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], ) + with TaskGroup( + "firefox_desktop_active_users_aggregates_external", + ) as firefox_desktop_active_users_aggregates_external: + ExternalTaskMarker( + task_id="bqetl_search_dashboard__wait_for_firefox_desktop_active_users_aggregates", + external_dag_id="bqetl_search_dashboard", + external_task_id="wait_for_firefox_desktop_active_users_aggregates", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", + ) + + firefox_desktop_active_users_aggregates_external.set_upstream( + firefox_desktop_active_users_aggregates + ) + firefox_ios_active_users_aggregates = bigquery_etl_query( task_id="firefox_ios_active_users_aggregates", destination_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_analytics_tables.py /tmp/workspace/generated-sql/dags/bqetl_analytics_tables.py --- /tmp/workspace/main-generated-sql/dags/bqetl_analytics_tables.py 2024-07-18 14:55:04.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_analytics_tables.py 2024-07-18 15:11:50.000000000 +0000 @@ -265,13 +265,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_feature_usage__wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", external_dag_id="bqetl_mobile_feature_usage", external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", @@ -331,6 +324,12 @@ ) ExternalTaskMarker( + task_id="bqetl_mozilla_org_derived__wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", + external_dag_id="bqetl_mozilla_org_derived", + external_task_id="wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", + ) + + ExternalTaskMarker( task_id="bqetl_google_analytics_derived_ga4__wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", external_dag_id="bqetl_google_analytics_derived_ga4", external_task_id="wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_download_funnel_attribution.py /tmp/workspace/generated-sql/dags/bqetl_download_funnel_attribution.py --- /tmp/workspace/main-generated-sql/dags/bqetl_download_funnel_attribution.py 2024-07-18 14:55:04.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_download_funnel_attribution.py 2024-07-18 15:11:50.000000000 +0000 @@ -75,19 +75,6 @@ retries=0, ) - checks__fail_mozilla_org_derived__downloads_with_attribution__v2 = bigquery_dq_check( - task_id="checks__fail_mozilla_org_derived__downloads_with_attribution__v2", - source_table='downloads_with_attribution_v2${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="mozilla_org_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=True, - owner="gleonard@mozilla.com", - email=["gleonard@mozilla.com", "telemetry-alerts@mozilla.com"], - depends_on_past=False, - parameters=["download_date:DATE:{{macros.ds_add(ds, -1)}}"], - retries=0, - ) - ga_derived__downloads_with_attribution__v2 = bigquery_etl_query( task_id="ga_derived__downloads_with_attribution__v2", destination_table='downloads_with_attribution_v2${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', @@ -100,26 +87,10 @@ parameters=["download_date:DATE:{{macros.ds_add(ds, -1)}}"], ) - mozilla_org_derived__downloads_with_attribution__v2 = bigquery_etl_query( - task_id="mozilla_org_derived__downloads_with_attribution__v2", - destination_table='downloads_with_attribution_v2${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="mozilla_org_derived", - project_id="moz-fx-data-shared-prod", - owner="gleonard@mozilla.com", - email=["gleonard@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["download_date:DATE:{{macros.ds_add(ds, -1)}}"], - ) - checks__fail_ga_derived__downloads_with_attribution__v2.set_upstream( ga_derived__downloads_with_attribution__v2 ) - checks__fail_mozilla_org_derived__downloads_with_attribution__v2.set_upstream( - mozilla_org_derived__downloads_with_attribution__v2 - ) - ga_derived__downloads_with_attribution__v2.set_upstream( wait_for_ga_derived__www_site_empty_check__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_firefox_desktop_ad_click_history.py /tmp/workspace/generated-sql/dags/bqetl_firefox_desktop_ad_click_history.py --- /tmp/workspace/main-generated-sql/dags/bqetl_firefox_desktop_ad_click_history.py 2024-07-18 14:55:04.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_firefox_desktop_ad_click_history.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,98 +0,0 @@ -# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py - -from airflow import DAG -from airflow.sensors.external_task import ExternalTaskMarker -from airflow.sensors.external_task import ExternalTaskSensor -from airflow.utils.task_group import TaskGroup -import datetime -from operators.gcp_container_operator import GKEPodOperator -from utils.constants import ALLOWED_STATES, FAILED_STATES -from utils.gcp import bigquery_etl_query, bigquery_dq_check - -docs = """ -### bqetl_firefox_desktop_ad_click_history - -Built from bigquery-etl repo, [`dags/bqetl_firefox_desktop_ad_click_history.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_firefox_desktop_ad_click_history.py) - -#### Description - -Calculates # of historical ad clicks for Firefox Desktop clients - -#### Owner - -kwindau@mozilla.com - -#### Tags - -* impact/tier_2 -* repo/bigquery-etl -""" - - -default_args = { - "owner": "kwindau@mozilla.com", - "start_date": datetime.datetime(2024, 7, 16, 0, 0), - "end_date": None, - "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], - "depends_on_past": False, - "retry_delay": datetime.timedelta(seconds=1800), - "email_on_failure": True, - "email_on_retry": False, - "retries": 2, -} - -tags = ["impact/tier_2", "repo/bigquery-etl"] - -with DAG( - "bqetl_firefox_desktop_ad_click_history", - default_args=default_args, - schedule_interval="0 16 * * *", - doc_md=docs, - tags=tags, -) as dag: - - wait_for_search_derived__search_clients_daily__v8 = ExternalTaskSensor( - task_id="wait_for_search_derived__search_clients_daily__v8", - external_dag_id="bqetl_search", - external_task_id="search_derived__search_clients_daily__v8", - execution_delta=datetime.timedelta(seconds=46800), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - checks__fail_firefox_desktop_derived__adclick_history__v1 = bigquery_dq_check( - task_id="checks__fail_firefox_desktop_derived__adclick_history__v1", - source_table="adclick_history_v1", - dataset_id="firefox_desktop_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=True, - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], - depends_on_past=False, - task_concurrency=1, - parameters=["submission_date:DATE:{{ds}}"], - retries=0, - ) - - firefox_desktop_derived__adclick_history__v1 = bigquery_etl_query( - task_id="firefox_desktop_derived__adclick_history__v1", - destination_table="adclick_history_v1", - dataset_id="firefox_desktop_derived", - project_id="moz-fx-data-shared-prod", - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=True, - parameters=["submission_date:DATE:{{ds}}"], - ) - - checks__fail_firefox_desktop_derived__adclick_history__v1.set_upstream( - firefox_desktop_derived__adclick_history__v1 - ) - - firefox_desktop_derived__adclick_history__v1.set_upstream( - wait_for_search_derived__search_clients_daily__v8 - ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_firefox_ios.py /tmp/workspace/generated-sql/dags/bqetl_firefox_ios.py --- /tmp/workspace/main-generated-sql/dags/bqetl_firefox_ios.py 2024-07-18 14:55:04.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_firefox_ios.py 2024-07-18 15:11:49.000000000 +0000 @@ -299,13 +299,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=57600)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_feature_usage__wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", external_dag_id="bqetl_mobile_feature_usage", external_task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_generated_funnels.py /tmp/workspace/generated-sql/dags/bqetl_generated_funnels.py --- /tmp/workspace/main-generated-sql/dags/bqetl_generated_funnels.py 2024-07-18 14:55:04.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_generated_funnels.py 2024-07-18 15:11:51.000000000 +0000 @@ -249,6 +249,21 @@ ) ) + monitor_frontend_derived__monitor_dashboard_user_journey_funnels__v1 = bigquery_etl_query( + task_id="monitor_frontend_derived__monitor_dashboard_user_journey_funnels__v1", + destination_table="monitor_dashboard_user_journey_funnels_v1", + dataset_id="monitor_frontend_derived", + project_id="moz-fx-data-shared-prod", + owner="ksiegler@mozilla.org", + email=[ + "ascholtz@mozilla.com", + "ksiegler@mozilla.org", + "telemetry-alerts@mozilla.com", + ], + date_partition_parameter="submission_date", + depends_on_past=False, + ) + accounts_frontend_derived__email_first_reg_login_funnels__v1.set_upstream( wait_for_copy_deduplicate_all ) @@ -294,3 +309,7 @@ firefox_accounts_derived__registration_funnels_legacy_events__v1.set_upstream( wait_for_firefox_accounts_derived__fxa_stdout_events__v1 ) + + monitor_frontend_derived__monitor_dashboard_user_journey_funnels__v1.set_upstream( + wait_for_copy_deduplicate_all + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived.py 2024-07-18 14:55:04.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived.py 2024-07-18 15:11:49.000000000 +0000 @@ -281,21 +281,6 @@ depends_on_past=False, ) - mozilla_org_derived__blogs_daily_summary__v1 = bigquery_etl_query( - task_id="mozilla_org_derived__blogs_daily_summary__v1", - destination_table="blogs_daily_summary_v1", - dataset_id="mozilla_org_derived", - project_id="moz-fx-data-shared-prod", - owner="ascholtz@mozilla.com", - email=[ - "ascholtz@mozilla.com", - "kwindau@mozilla.com", - "telemetry-alerts@mozilla.com", - ], - date_partition_parameter="submission_date", - depends_on_past=False, - ) - mozilla_org_derived__blogs_goals__v1 = bigquery_etl_query( task_id="mozilla_org_derived__blogs_goals__v1", destination_table="blogs_goals_v1", @@ -311,21 +296,6 @@ depends_on_past=False, ) - mozilla_org_derived__blogs_landing_page_summary__v1 = bigquery_etl_query( - task_id="mozilla_org_derived__blogs_landing_page_summary__v1", - destination_table="blogs_landing_page_summary_v1", - dataset_id="mozilla_org_derived", - project_id="moz-fx-data-shared-prod", - owner="ascholtz@mozilla.com", - email=[ - "ascholtz@mozilla.com", - "kwindau@mozilla.com", - "telemetry-alerts@mozilla.com", - ], - date_partition_parameter="submission_date", - depends_on_past=False, - ) - mozilla_org_derived__blogs_sessions__v1 = bigquery_etl_query( task_id="mozilla_org_derived__blogs_sessions__v1", destination_table="blogs_sessions_v1", @@ -341,21 +311,6 @@ depends_on_past=False, ) - mozilla_org_derived__downloads_with_attribution__v1 = bigquery_etl_query( - task_id="mozilla_org_derived__downloads_with_attribution__v1", - destination_table="downloads_with_attribution_v1", - dataset_id="mozilla_org_derived", - project_id="moz-fx-data-shared-prod", - owner="gleonard@mozilla.com", - email=[ - "gleonard@mozilla.com", - "kwindau@mozilla.com", - "telemetry-alerts@mozilla.com", - ], - date_partition_parameter="submission_date", - depends_on_past=False, - ) - mozilla_org_derived__firefox_whatsnew_summary__v1 = bigquery_etl_query( task_id="mozilla_org_derived__firefox_whatsnew_summary__v1", destination_table="firefox_whatsnew_summary_v1", @@ -497,22 +452,6 @@ ga_derived__www_site_page_metrics__v1.set_upstream(ga_derived__www_site_hits__v1) - mozilla_org_derived__blogs_daily_summary__v1.set_upstream( - mozilla_org_derived__blogs_goals__v1 - ) - - mozilla_org_derived__blogs_daily_summary__v1.set_upstream( - mozilla_org_derived__blogs_sessions__v1 - ) - - mozilla_org_derived__blogs_landing_page_summary__v1.set_upstream( - mozilla_org_derived__blogs_goals__v1 - ) - - mozilla_org_derived__blogs_landing_page_summary__v1.set_upstream( - mozilla_org_derived__blogs_sessions__v1 - ) - mozilla_org_derived__firefox_whatsnew_summary__v1.set_upstream( mozilla_org_derived__www_site_hits__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_main_summary.py /tmp/workspace/generated-sql/dags/bqetl_main_summary.py --- /tmp/workspace/main-generated-sql/dags/bqetl_main_summary.py 2024-07-18 14:55:04.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_main_summary.py 2024-07-18 15:11:48.000000000 +0000 @@ -339,6 +339,12 @@ ) ExternalTaskMarker( + task_id="bqetl_mozilla_org_derived__wait_for_telemetry_derived__clients_daily__v6", + external_dag_id="bqetl_mozilla_org_derived", + external_task_id="wait_for_telemetry_derived__clients_daily__v6", + ) + + ExternalTaskMarker( task_id="bqetl_google_analytics_derived_ga4__wait_for_telemetry_derived__clients_daily__v6", external_dag_id="bqetl_google_analytics_derived_ga4", external_task_id="wait_for_telemetry_derived__clients_daily__v6", diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_mobile_kpi_metrics.py /tmp/workspace/generated-sql/dags/bqetl_mobile_kpi_metrics.py --- /tmp/workspace/main-generated-sql/dags/bqetl_mobile_kpi_metrics.py 2024-07-18 14:55:04.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_mobile_kpi_metrics.py 2024-07-18 15:11:52.000000000 +0000 @@ -62,20 +62,6 @@ task_group_klar_ios = TaskGroup("klar_ios") - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 = ( - ExternalTaskSensor( - task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - external_dag_id="bqetl_analytics_tables", - external_task_id="checks__fail_fenix_derived__firefox_android_clients__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - ) - wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_glean_usage", @@ -136,6 +122,18 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) + wait_for_copy_deduplicate_all = ExternalTaskSensor( + task_id="wait_for_copy_deduplicate_all", + external_dag_id="copy_deduplicate", + external_task_id="copy_deduplicate_all", + execution_delta=datetime.timedelta(seconds=39600), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1 = ExternalTaskSensor( task_id="wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", external_dag_id="bqetl_glean_usage", @@ -196,18 +194,6 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - external_dag_id="bqetl_firefox_ios", - external_task_id="checks__fail_firefox_ios_derived__firefox_ios_clients__v1", - execution_delta=datetime.timedelta(seconds=28800), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_glean_usage", @@ -424,6 +410,20 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) + checks__warn_fenix_derived__attribution_clients__v1 = bigquery_dq_check( + task_id="checks__warn_fenix_derived__attribution_clients__v1", + source_table="attribution_clients_v1", + dataset_id="fenix_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=False, + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_fenix, + ) + checks__warn_fenix_derived__engagement__v1 = bigquery_dq_check( task_id="checks__warn_fenix_derived__engagement__v1", source_table="engagement_v1", @@ -453,6 +453,20 @@ task_group=task_group_fenix, ) + checks__warn_firefox_ios_derived__attribution_clients__v1 = bigquery_dq_check( + task_id="checks__warn_firefox_ios_derived__attribution_clients__v1", + source_table="attribution_clients_v1", + dataset_id="firefox_ios_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=False, + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_firefox_ios, + ) + checks__warn_firefox_ios_derived__engagement__v1 = bigquery_dq_check( task_id="checks__warn_firefox_ios_derived__engagement__v1", source_table="engagement_v1", @@ -482,6 +496,20 @@ task_group=task_group_firefox_ios, ) + checks__warn_focus_android_derived__attribution_clients__v1 = bigquery_dq_check( + task_id="checks__warn_focus_android_derived__attribution_clients__v1", + source_table="attribution_clients_v1", + dataset_id="focus_android_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=False, + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_focus_android, + ) + checks__warn_focus_android_derived__engagement__v1 = bigquery_dq_check( task_id="checks__warn_focus_android_derived__engagement__v1", source_table="engagement_v1", @@ -511,6 +539,20 @@ task_group=task_group_focus_android, ) + checks__warn_focus_ios_derived__attribution_clients__v1 = bigquery_dq_check( + task_id="checks__warn_focus_ios_derived__attribution_clients__v1", + source_table="attribution_clients_v1", + dataset_id="focus_ios_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=False, + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_focus_ios, + ) + checks__warn_focus_ios_derived__engagement__v1 = bigquery_dq_check( task_id="checks__warn_focus_ios_derived__engagement__v1", source_table="engagement_v1", @@ -540,6 +582,20 @@ task_group=task_group_focus_ios, ) + checks__warn_klar_android_derived__attribution_clients__v1 = bigquery_dq_check( + task_id="checks__warn_klar_android_derived__attribution_clients__v1", + source_table="attribution_clients_v1", + dataset_id="klar_android_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=False, + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_klar_android, + ) + checks__warn_klar_android_derived__engagement__v1 = bigquery_dq_check( task_id="checks__warn_klar_android_derived__engagement__v1", source_table="engagement_v1", @@ -569,6 +625,20 @@ task_group=task_group_klar_android, ) + checks__warn_klar_ios_derived__attribution_clients__v1 = bigquery_dq_check( + task_id="checks__warn_klar_ios_derived__attribution_clients__v1", + source_table="attribution_clients_v1", + dataset_id="klar_ios_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=False, + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_klar_ios, + ) + checks__warn_klar_ios_derived__engagement__v1 = bigquery_dq_check( task_id="checks__warn_klar_ios_derived__engagement__v1", source_table="engagement_v1", @@ -598,6 +668,18 @@ task_group=task_group_klar_ios, ) + fenix_derived__attribution_clients__v1 = bigquery_etl_query( + task_id="fenix_derived__attribution_clients__v1", + destination_table="attribution_clients_v1", + dataset_id="fenix_derived", + project_id="moz-fx-data-shared-prod", + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + task_group=task_group_fenix, + ) + fenix_derived__engagement__v1 = bigquery_etl_query( task_id="fenix_derived__engagement__v1", destination_table="engagement_v1", @@ -610,6 +692,18 @@ task_group=task_group_fenix, ) + fenix_derived__new_profiles__v1 = bigquery_etl_query( + task_id="fenix_derived__new_profiles__v1", + destination_table="new_profiles_v1", + dataset_id="fenix_derived", + project_id="moz-fx-data-shared-prod", + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + task_group=task_group_fenix, + ) + fenix_derived__retention__v1 = bigquery_etl_query( task_id="fenix_derived__retention__v1", destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', @@ -624,6 +718,18 @@ task_group=task_group_fenix, ) + firefox_ios_derived__attribution_clients__v1 = bigquery_etl_query( + task_id="firefox_ios_derived__attribution_clients__v1", + destination_table="attribution_clients_v1", + dataset_id="firefox_ios_derived", + project_id="moz-fx-data-shared-prod", + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + task_group=task_group_firefox_ios, + ) + firefox_ios_derived__engagement__v1 = bigquery_etl_query( task_id="firefox_ios_derived__engagement__v1", destination_table="engagement_v1", @@ -636,6 +742,18 @@ task_group=task_group_firefox_ios, ) + firefox_ios_derived__new_profiles__v1 = bigquery_etl_query( + task_id="firefox_ios_derived__new_profiles__v1", + destination_table="new_profiles_v1", + dataset_id="firefox_ios_derived", + project_id="moz-fx-data-shared-prod", + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + task_group=task_group_firefox_ios, + ) + firefox_ios_derived__retention__v1 = bigquery_etl_query( task_id="firefox_ios_derived__retention__v1", destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', @@ -650,6 +768,18 @@ task_group=task_group_firefox_ios, ) + focus_android_derived__attribution_clients__v1 = bigquery_etl_query( + task_id="focus_android_derived__attribution_clients__v1", + destination_table="attribution_clients_v1", + dataset_id="focus_android_derived", + project_id="moz-fx-data-shared-prod", + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + task_group=task_group_focus_android, + ) + focus_android_derived__engagement__v1 = bigquery_etl_query( task_id="focus_android_derived__engagement__v1", destination_table="engagement_v1", @@ -662,6 +792,18 @@ task_group=task_group_focus_android, ) + focus_android_derived__new_profiles__v1 = bigquery_etl_query( + task_id="focus_android_derived__new_profiles__v1", + destination_table="new_profiles_v1", + dataset_id="focus_android_derived", + project_id="moz-fx-data-shared-prod", + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + task_group=task_group_focus_android, + ) + focus_android_derived__retention__v1 = bigquery_etl_query( task_id="focus_android_derived__retention__v1", destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', @@ -676,6 +818,18 @@ task_group=task_group_focus_android, ) + focus_ios_derived__attribution_clients__v1 = bigquery_etl_query( + task_id="focus_ios_derived__attribution_clients__v1", + destination_table="attribution_clients_v1", + dataset_id="focus_ios_derived", + project_id="moz-fx-data-shared-prod", + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + task_group=task_group_focus_ios, + ) + focus_ios_derived__engagement__v1 = bigquery_etl_query( task_id="focus_ios_derived__engagement__v1", destination_table="engagement_v1", @@ -688,6 +842,18 @@ task_group=task_group_focus_ios, ) + focus_ios_derived__new_profiles__v1 = bigquery_etl_query( + task_id="focus_ios_derived__new_profiles__v1", + destination_table="new_profiles_v1", + dataset_id="focus_ios_derived", + project_id="moz-fx-data-shared-prod", + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + task_group=task_group_focus_ios, + ) + focus_ios_derived__retention__v1 = bigquery_etl_query( task_id="focus_ios_derived__retention__v1", destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', @@ -702,6 +868,18 @@ task_group=task_group_focus_ios, ) + klar_android_derived__attribution_clients__v1 = bigquery_etl_query( + task_id="klar_android_derived__attribution_clients__v1", + destination_table="attribution_clients_v1", + dataset_id="klar_android_derived", + project_id="moz-fx-data-shared-prod", + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + task_group=task_group_klar_android, + ) + klar_android_derived__engagement__v1 = bigquery_etl_query( task_id="klar_android_derived__engagement__v1", destination_table="engagement_v1", @@ -714,6 +892,18 @@ task_group=task_group_klar_android, ) + klar_android_derived__new_profiles__v1 = bigquery_etl_query( + task_id="klar_android_derived__new_profiles__v1", + destination_table="new_profiles_v1", + dataset_id="klar_android_derived", + project_id="moz-fx-data-shared-prod", + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + task_group=task_group_klar_android, + ) + klar_android_derived__retention__v1 = bigquery_etl_query( task_id="klar_android_derived__retention__v1", destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', @@ -728,6 +918,18 @@ task_group=task_group_klar_android, ) + klar_ios_derived__attribution_clients__v1 = bigquery_etl_query( + task_id="klar_ios_derived__attribution_clients__v1", + destination_table="attribution_clients_v1", + dataset_id="klar_ios_derived", + project_id="moz-fx-data-shared-prod", + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + task_group=task_group_klar_ios, + ) + klar_ios_derived__engagement__v1 = bigquery_etl_query( task_id="klar_ios_derived__engagement__v1", destination_table="engagement_v1", @@ -740,6 +942,18 @@ task_group=task_group_klar_ios, ) + klar_ios_derived__new_profiles__v1 = bigquery_etl_query( + task_id="klar_ios_derived__new_profiles__v1", + destination_table="new_profiles_v1", + dataset_id="klar_ios_derived", + project_id="moz-fx-data-shared-prod", + owner="mozilla/kpi_table_reviewers", + email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + task_group=task_group_klar_ios, + ) + klar_ios_derived__retention__v1 = bigquery_etl_query( task_id="klar_ios_derived__retention__v1", destination_table='retention_v1${{ macros.ds_format(macros.ds_add(ds, -27), "%Y-%m-%d", "%Y%m%d") }}', @@ -754,12 +968,20 @@ task_group=task_group_klar_ios, ) + checks__warn_fenix_derived__attribution_clients__v1.set_upstream( + fenix_derived__attribution_clients__v1 + ) + checks__warn_fenix_derived__engagement__v1.set_upstream( fenix_derived__engagement__v1 ) checks__warn_fenix_derived__retention__v1.set_upstream(fenix_derived__retention__v1) + checks__warn_firefox_ios_derived__attribution_clients__v1.set_upstream( + firefox_ios_derived__attribution_clients__v1 + ) + checks__warn_firefox_ios_derived__engagement__v1.set_upstream( firefox_ios_derived__engagement__v1 ) @@ -768,6 +990,10 @@ firefox_ios_derived__retention__v1 ) + checks__warn_focus_android_derived__attribution_clients__v1.set_upstream( + focus_android_derived__attribution_clients__v1 + ) + checks__warn_focus_android_derived__engagement__v1.set_upstream( focus_android_derived__engagement__v1 ) @@ -776,6 +1002,10 @@ focus_android_derived__retention__v1 ) + checks__warn_focus_ios_derived__attribution_clients__v1.set_upstream( + focus_ios_derived__attribution_clients__v1 + ) + checks__warn_focus_ios_derived__engagement__v1.set_upstream( focus_ios_derived__engagement__v1 ) @@ -784,6 +1014,10 @@ focus_ios_derived__retention__v1 ) + checks__warn_klar_android_derived__attribution_clients__v1.set_upstream( + klar_android_derived__attribution_clients__v1 + ) + checks__warn_klar_android_derived__engagement__v1.set_upstream( klar_android_derived__engagement__v1 ) @@ -792,6 +1026,10 @@ klar_android_derived__retention__v1 ) + checks__warn_klar_ios_derived__attribution_clients__v1.set_upstream( + klar_ios_derived__attribution_clients__v1 + ) + checks__warn_klar_ios_derived__engagement__v1.set_upstream( klar_ios_derived__engagement__v1 ) @@ -800,10 +1038,28 @@ klar_ios_derived__retention__v1 ) - fenix_derived__engagement__v1.set_upstream( - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 + fenix_derived__attribution_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 + ) + + fenix_derived__attribution_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 + ) + + fenix_derived__attribution_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 + ) + + fenix_derived__attribution_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 + ) + + fenix_derived__attribution_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 ) + fenix_derived__attribution_clients__v1.set_upstream(wait_for_copy_deduplicate_all) + fenix_derived__engagement__v1.set_upstream( wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 ) @@ -824,8 +1080,24 @@ wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 ) - fenix_derived__retention__v1.set_upstream( - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 + fenix_derived__new_profiles__v1.set_upstream( + wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 + ) + + fenix_derived__new_profiles__v1.set_upstream( + wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 + ) + + fenix_derived__new_profiles__v1.set_upstream( + wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 + ) + + fenix_derived__new_profiles__v1.set_upstream( + wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 + ) + + fenix_derived__new_profiles__v1.set_upstream( + wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 ) fenix_derived__retention__v1.set_upstream( @@ -868,8 +1140,20 @@ wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1 ) - firefox_ios_derived__engagement__v1.set_upstream( - wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 + firefox_ios_derived__attribution_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 + ) + + firefox_ios_derived__attribution_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + firefox_ios_derived__attribution_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 + ) + + firefox_ios_derived__attribution_clients__v1.set_upstream( + wait_for_copy_deduplicate_all ) firefox_ios_derived__engagement__v1.set_upstream( @@ -884,8 +1168,16 @@ wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 ) - firefox_ios_derived__retention__v1.set_upstream( - wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 + firefox_ios_derived__new_profiles__v1.set_upstream( + wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 + ) + + firefox_ios_derived__new_profiles__v1.set_upstream( + wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + firefox_ios_derived__new_profiles__v1.set_upstream( + wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 ) firefox_ios_derived__retention__v1.set_upstream( @@ -912,6 +1204,22 @@ wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 ) + focus_android_derived__attribution_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 + ) + + focus_android_derived__attribution_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 + ) + + focus_android_derived__attribution_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 + ) + + focus_android_derived__attribution_clients__v1.set_upstream( + wait_for_copy_deduplicate_all + ) + focus_android_derived__engagement__v1.set_upstream( wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 ) @@ -924,6 +1232,18 @@ wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 ) + focus_android_derived__new_profiles__v1.set_upstream( + wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 + ) + + focus_android_derived__new_profiles__v1.set_upstream( + wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 + ) + + focus_android_derived__new_profiles__v1.set_upstream( + wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 + ) + focus_android_derived__retention__v1.set_upstream( wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 ) @@ -948,10 +1268,18 @@ wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1 ) + focus_ios_derived__attribution_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 + ) + focus_ios_derived__engagement__v1.set_upstream( wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 ) + focus_ios_derived__new_profiles__v1.set_upstream( + wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 + ) + focus_ios_derived__retention__v1.set_upstream( wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 ) @@ -960,10 +1288,22 @@ wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1 ) + klar_android_derived__attribution_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1 + ) + + klar_android_derived__attribution_clients__v1.set_upstream( + wait_for_copy_deduplicate_all + ) + klar_android_derived__engagement__v1.set_upstream( wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1 ) + klar_android_derived__new_profiles__v1.set_upstream( + wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1 + ) + klar_android_derived__retention__v1.set_upstream( wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1 ) @@ -972,10 +1312,18 @@ wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1 ) + klar_ios_derived__attribution_clients__v1.set_upstream( + wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1 + ) + klar_ios_derived__engagement__v1.set_upstream( wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1 ) + klar_ios_derived__new_profiles__v1.set_upstream( + wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1 + ) + klar_ios_derived__retention__v1.set_upstream( wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_mozilla_org_derived.py /tmp/workspace/generated-sql/dags/bqetl_mozilla_org_derived.py --- /tmp/workspace/main-generated-sql/dags/bqetl_mozilla_org_derived.py 2024-07-18 14:55:04.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_mozilla_org_derived.py 2024-07-18 15:11:51.000000000 +0000 @@ -47,6 +47,106 @@ tags=tags, ) as dag: + wait_for_mozilla_org_derived__ga_sessions__v1__backfill__1 = ExternalTaskSensor( + task_id="wait_for_mozilla_org_derived__ga_sessions__v1__backfill_-1", + external_dag_id="ga_sessions_backfill", + external_task_id="mozilla_org_derived__ga_sessions__v1__backfill_-1", + execution_delta=datetime.timedelta(seconds=3600), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_mozilla_org_derived__ga_sessions__v1__backfill__2 = ExternalTaskSensor( + task_id="wait_for_mozilla_org_derived__ga_sessions__v1__backfill_-2", + external_dag_id="ga_sessions_backfill", + external_task_id="mozilla_org_derived__ga_sessions__v1__backfill_-2", + execution_delta=datetime.timedelta(seconds=3600), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_mozilla_org_derived__ga_sessions__v1__backfill__3 = ExternalTaskSensor( + task_id="wait_for_mozilla_org_derived__ga_sessions__v1__backfill_-3", + external_dag_id="ga_sessions_backfill", + external_task_id="mozilla_org_derived__ga_sessions__v1__backfill_-3", + execution_delta=datetime.timedelta(seconds=3600), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_checks__fail_telemetry_derived__clients_first_seen__v2 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", + external_dag_id="bqetl_analytics_tables", + external_task_id="checks__fail_telemetry_derived__clients_first_seen__v2", + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + wait_for_telemetry_derived__clients_daily__v6 = ExternalTaskSensor( + task_id="wait_for_telemetry_derived__clients_daily__v6", + external_dag_id="bqetl_main_summary", + external_task_id="telemetry_derived__clients_daily__v6", + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + checks__fail_mozilla_org_derived__ga_clients__v1 = bigquery_dq_check( + task_id="checks__fail_mozilla_org_derived__ga_clients__v1", + source_table="ga_clients_v1", + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=True, + owner="frank@mozilla.com", + email=["frank@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + task_concurrency=1, + parameters=["session_date:DATE:{{ds}}"], + retries=0, + ) + + checks__fail_mozilla_org_derived__ga_sessions__v1 = bigquery_dq_check( + task_id="checks__fail_mozilla_org_derived__ga_sessions__v1", + source_table="ga_sessions_v1", + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=True, + owner="frank@mozilla.com", + email=["frank@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["session_date:DATE:{{ds}}"], + retries=0, + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v1 = bigquery_dq_check( + task_id="checks__fail_mozilla_org_derived__gclid_conversions__v1", + source_table="gclid_conversions_v1", + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=True, + owner="frank@mozilla.com", + email=["frank@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["conversion_window:INT64:30"] + ["activity_date:DATE:{{ds}}"], + retries=0, + ) + checks__fail_stub_attribution_service_derived__dl_token_ga_attribution_lookup__v1 = bigquery_dq_check( task_id="checks__fail_stub_attribution_service_derived__dl_token_ga_attribution_lookup__v1", source_table="dl_token_ga_attribution_lookup_v1", @@ -75,6 +175,41 @@ checks__fail_stub_attribution_service_derived__dl_token_ga_attribution_lookup__v1 ) + mozilla_org_derived__ga_clients__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_clients__v1", + destination_table="ga_clients_v1", + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="frank@mozilla.com", + email=["frank@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=True, + parameters=["session_date:DATE:{{ds}}"], + ) + + mozilla_org_derived__ga_sessions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_sessions__v1", + destination_table="ga_sessions_v1", + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="frank@mozilla.com", + email=["frank@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="session_date", + depends_on_past=False, + ) + + mozilla_org_derived__gclid_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__gclid_conversions__v1", + destination_table="gclid_conversions_v1", + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="frank@mozilla.com", + email=["frank@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="activity_date", + depends_on_past=False, + parameters=["conversion_window:INT64:30"], + ) + stub_attribution_service_derived__dl_token_ga_attribution_lookup__v1 = bigquery_etl_query( task_id="stub_attribution_service_derived__dl_token_ga_attribution_lookup__v1", destination_table="dl_token_ga_attribution_lookup_v1", @@ -88,6 +223,50 @@ parameters=["download_date:DATE:{{ds}}"], ) + checks__fail_mozilla_org_derived__ga_clients__v1.set_upstream( + mozilla_org_derived__ga_clients__v1 + ) + + checks__fail_mozilla_org_derived__ga_sessions__v1.set_upstream( + mozilla_org_derived__ga_sessions__v1 + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v1.set_upstream( + mozilla_org_derived__gclid_conversions__v1 + ) + checks__fail_stub_attribution_service_derived__dl_token_ga_attribution_lookup__v1.set_upstream( stub_attribution_service_derived__dl_token_ga_attribution_lookup__v1 ) + + mozilla_org_derived__ga_clients__v1.set_upstream( + checks__fail_mozilla_org_derived__ga_sessions__v1 + ) + + mozilla_org_derived__ga_sessions__v1.set_upstream( + wait_for_mozilla_org_derived__ga_sessions__v1__backfill__1 + ) + + mozilla_org_derived__ga_sessions__v1.set_upstream( + wait_for_mozilla_org_derived__ga_sessions__v1__backfill__2 + ) + + mozilla_org_derived__ga_sessions__v1.set_upstream( + wait_for_mozilla_org_derived__ga_sessions__v1__backfill__3 + ) + + mozilla_org_derived__gclid_conversions__v1.set_upstream( + checks__fail_mozilla_org_derived__ga_sessions__v1 + ) + + mozilla_org_derived__gclid_conversions__v1.set_upstream( + checks__fail_stub_attribution_service_derived__dl_token_ga_attribution_lookup__v1 + ) + + mozilla_org_derived__gclid_conversions__v1.set_upstream( + wait_for_checks__fail_telemetry_derived__clients_first_seen__v2 + ) + + mozilla_org_derived__gclid_conversions__v1.set_upstream( + wait_for_telemetry_derived__clients_daily__v6 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_search_dashboard.py /tmp/workspace/generated-sql/dags/bqetl_search_dashboard.py --- /tmp/workspace/main-generated-sql/dags/bqetl_search_dashboard.py 2024-07-18 14:55:04.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_search_dashboard.py 2024-07-18 15:11:49.000000000 +0000 @@ -83,119 +83,25 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), + wait_for_checks__fail_fenix_derived__active_users_aggregates__v3 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_fenix_derived__active_users_aggregates__v3", + external_dag_id="bqetl_analytics_aggregations", + external_task_id="checks__fail_fenix_derived__active_users_aggregates__v3", + execution_delta=datetime.timedelta(seconds=900), check_existence=True, mode="reschedule", allowed_states=ALLOWED_STATES, failed_states=FAILED_STATES, pool="DATA_ENG_EXTERNALTASKSENSOR", ) - - wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_nightly_derived__baselin ```⚠️ Only part of the diff is displayed.
Note for the reviewer:
sql_generators/mobile_kpi_support_metrics/gen_sql/moz-fx-data-shared-prod/...path will be deleted prior merging, it was included only to show how the generation modified the queries: https://github.com/mozilla/bigquery-etl/pull/5672/commits/e255d6b21bdac9d54a8b08439fd617b0acc4f7a2#diff-8f7196d7fe625d081c409a80b1fd1a56e13e84f2d0bd779e4da34b0d56cd78bffeat(): Add new_profiles and attribution to mobile_kpi_support_metrics generator
This change introduces the following new artifacts to each mobile product (used in this generator):
New profiles - responsible for enabling reporting around how many new clients / profiles each product has which can be broken down by dimensions used for our current KPI reporting (such as country, channel).
Attribution - aims to act as the source of truth for identifying how to attribute a profile / client.
Checklist for reviewer:
<username>:<branch>of the fork as parameter. The parameter will also show up in the logs of themanual-trigger-required-for-forkCI task together with more detailed instructions.For modifications to schemas in restricted namespaces (see
CODEOWNERS):┆Issue is synchronized with this Jira Task