dataops-ci-bot
commented
2 months ago Integration report for "re-add troubleshooting step"
sql.diff
Click to expand!
```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_accounts_backend_external.py /tmp/workspace/generated-sql/dags/bqetl_accounts_backend_external.py --- /tmp/workspace/main-generated-sql/dags/bqetl_accounts_backend_external.py 2024-08-02 21:40:12.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_accounts_backend_external.py 2024-08-02 21:38:01.000000000 +0000 @@ -124,18 +124,6 @@ task_concurrency=1, ) - accounts_db_external__fxa_account_groups__v1 = bigquery_etl_query( - task_id="accounts_db_external__fxa_account_groups__v1", - destination_table="fxa_account_groups_v1", - dataset_id="accounts_db_external", - project_id="moz-fx-data-shared-prod", - owner="akomar@mozilla.com", - email=["akomar@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - task_concurrency=1, - ) - accounts_db_external__fxa_account_reset_tokens__v1 = bigquery_etl_query( task_id="accounts_db_external__fxa_account_reset_tokens__v1", destination_table="fxa_account_reset_tokens_v1", @@ -160,18 +148,6 @@ task_concurrency=1, ) - accounts_db_external__fxa_carts__v1 = bigquery_etl_query( - task_id="accounts_db_external__fxa_carts__v1", - destination_table="fxa_carts_v1", - dataset_id="accounts_db_external", - project_id="moz-fx-data-shared-prod", - owner="akomar@mozilla.com", - email=["akomar@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - task_concurrency=1, - ) - accounts_db_external__fxa_db_metadata__v1 = bigquery_etl_query( task_id="accounts_db_external__fxa_db_metadata__v1", destination_table="fxa_db_metadata_v1", @@ -256,18 +232,6 @@ task_concurrency=1, ) - accounts_db_external__fxa_groups__v1 = bigquery_etl_query( - task_id="accounts_db_external__fxa_groups__v1", - destination_table="fxa_groups_v1", - dataset_id="accounts_db_external", - project_id="moz-fx-data-shared-prod", - owner="akomar@mozilla.com", - email=["akomar@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - task_concurrency=1, - ) - accounts_db_external__fxa_key_fetch_tokens__v1 = bigquery_etl_query( task_id="accounts_db_external__fxa_key_fetch_tokens__v1", destination_table="fxa_key_fetch_tokens_v1", @@ -604,18 +568,6 @@ task_concurrency=1, ) - accounts_db_nonprod_external__fxa_account_groups__v1 = bigquery_etl_query( - task_id="accounts_db_nonprod_external__fxa_account_groups__v1", - destination_table="fxa_account_groups_v1", - dataset_id="accounts_db_nonprod_external", - project_id="moz-fx-data-shared-prod", - owner="akomar@mozilla.com", - email=["akomar@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - task_concurrency=1, - ) - accounts_db_nonprod_external__fxa_account_reset_tokens__v1 = bigquery_etl_query( task_id="accounts_db_nonprod_external__fxa_account_reset_tokens__v1", destination_table="fxa_account_reset_tokens_v1", @@ -640,18 +592,6 @@ task_concurrency=1, ) - accounts_db_nonprod_external__fxa_carts__v1 = bigquery_etl_query( - task_id="accounts_db_nonprod_external__fxa_carts__v1", - destination_table="fxa_carts_v1", - dataset_id="accounts_db_nonprod_external", - project_id="moz-fx-data-shared-prod", - owner="akomar@mozilla.com", - email=["akomar@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - task_concurrency=1, - ) - accounts_db_nonprod_external__fxa_db_metadata__v1 = bigquery_etl_query( task_id="accounts_db_nonprod_external__fxa_db_metadata__v1", destination_table="fxa_db_metadata_v1", @@ -732,18 +672,6 @@ dataset_id="accounts_db_nonprod_external", project_id="moz-fx-data-shared-prod", owner="akomar@mozilla.com", - email=["akomar@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - task_concurrency=1, - ) - - accounts_db_nonprod_external__fxa_groups__v1 = bigquery_etl_query( - task_id="accounts_db_nonprod_external__fxa_groups__v1", - destination_table="fxa_groups_v1", - dataset_id="accounts_db_nonprod_external", - project_id="moz-fx-data-shared-prod", - owner="akomar@mozilla.com", email=["akomar@mozilla.com", "telemetry-alerts@mozilla.com"], date_partition_parameter=None, depends_on_past=False, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare_browser_market_share.py /tmp/workspace/generated-sql/dags/bqetl_cloudflare_browser_market_share.py --- /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare_browser_market_share.py 2024-08-02 21:40:12.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_cloudflare_browser_market_share.py 2024-08-02 21:38:03.000000000 +0000 @@ -51,25 +51,6 @@ tags=tags, ) as dag: - checks__warn_cloudflare_derived__browser_usage__v1 = bigquery_dq_check( - task_id="checks__warn_cloudflare_derived__browser_usage__v1", - source_table="browser_usage_v1", - dataset_id="cloudflare_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com"], - depends_on_past=False, - arguments=[ - "--date", - "{{ds}}", - "--cloudflare_api_token", - "{{ var.value.cloudflare_auth_token}}", - ], - parameters=["dte:DATE:{{ds}}"], - retries=0, - ) - cloudflare_derived__browser_usage__v1 = GKEPodOperator( task_id="cloudflare_derived__browser_usage__v1", arguments=[ @@ -86,7 +67,3 @@ owner="kwindau@mozilla.com", email=["kwindau@mozilla.com"], ) - - checks__warn_cloudflare_derived__browser_usage__v1.set_upstream( - cloudflare_derived__browser_usage__v1 - ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare_device_market_share.py /tmp/workspace/generated-sql/dags/bqetl_cloudflare_device_market_share.py --- /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare_device_market_share.py 2024-08-02 21:40:12.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_cloudflare_device_market_share.py 2024-08-02 21:38:03.000000000 +0000 @@ -51,25 +51,6 @@ tags=tags, ) as dag: - checks__warn_cloudflare_derived__device_usage__v1 = bigquery_dq_check( - task_id="checks__warn_cloudflare_derived__device_usage__v1", - source_table="device_usage_v1", - dataset_id="cloudflare_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com"], - depends_on_past=False, - arguments=[ - "--date", - "{{ds}}", - "--cloudflare_api_token", - "{{ var.value.cloudflare_auth_token}}", - ], - parameters=["dte:DATE:{{ds}}"], - retries=0, - ) - cloudflare_derived__device_usage__v1 = GKEPodOperator( task_id="cloudflare_derived__device_usage__v1", arguments=[ @@ -86,7 +67,3 @@ owner="kwindau@mozilla.com", email=["kwindau@mozilla.com"], ) - - checks__warn_cloudflare_derived__device_usage__v1.set_upstream( - cloudflare_derived__device_usage__v1 - ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare_os_market_share.py /tmp/workspace/generated-sql/dags/bqetl_cloudflare_os_market_share.py --- /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare_os_market_share.py 2024-08-02 21:40:12.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_cloudflare_os_market_share.py 2024-08-02 21:38:03.000000000 +0000 @@ -51,25 +51,6 @@ tags=tags, ) as dag: - checks__warn_cloudflare_derived__os_usage__v1 = bigquery_dq_check( - task_id="checks__warn_cloudflare_derived__os_usage__v1", - source_table="os_usage_v1", - dataset_id="cloudflare_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com"], - depends_on_past=False, - arguments=[ - "--date", - "{{ds}}", - "--cloudflare_api_token", - "{{ var.value.cloudflare_auth_token}}", - ], - parameters=["dte:DATE:{{ds}}"], - retries=0, - ) - cloudflare_derived__os_usage__v1 = GKEPodOperator( task_id="cloudflare_derived__os_usage__v1", arguments=[ @@ -86,7 +67,3 @@ owner="kwindau@mozilla.com", email=["kwindau@mozilla.com"], ) - - checks__warn_cloudflare_derived__os_usage__v1.set_upstream( - cloudflare_derived__os_usage__v1 - ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_generated_funnels.py /tmp/workspace/generated-sql/dags/bqetl_generated_funnels.py --- /tmp/workspace/main-generated-sql/dags/bqetl_generated_funnels.py 2024-08-02 21:40:12.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_generated_funnels.py 2024-08-02 21:38:03.000000000 +0000 @@ -75,18 +75,6 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_accounts_backend_derived__events_stream__v1 = ExternalTaskSensor( - task_id="wait_for_accounts_backend_derived__events_stream__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="accounts_backend.accounts_backend_derived__events_stream__v1", - execution_delta=datetime.timedelta(seconds=10800), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 = ( ExternalTaskSensor( task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", @@ -149,6 +137,21 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) + accounts_frontend_derived__email_first_reg_login_funnels__v1 = bigquery_etl_query( + task_id="accounts_frontend_derived__email_first_reg_login_funnels__v1", + destination_table="email_first_reg_login_funnels_v1", + dataset_id="accounts_frontend_derived", + project_id="moz-fx-data-shared-prod", + owner="ksiegler@mozilla.org", + email=[ + "ascholtz@mozilla.com", + "ksiegler@mozilla.org", + "telemetry-alerts@mozilla.com", + ], + date_partition_parameter="submission_date", + depends_on_past=False, + ) + accounts_frontend_derived__email_first_reg_login_funnels_by_service__v1 = bigquery_etl_query( task_id="accounts_frontend_derived__email_first_reg_login_funnels_by_service__v1", destination_table="email_first_reg_login_funnels_by_service_v1", @@ -303,19 +306,15 @@ depends_on_past=False, ) - accounts_frontend_derived__email_first_reg_login_funnels_by_service__v1.set_upstream( + accounts_frontend_derived__email_first_reg_login_funnels__v1.set_upstream( wait_for_copy_deduplicate_all ) - accounts_frontend_derived__login_engagement_funnel__v1.set_upstream( - wait_for_accounts_frontend_derived__events_stream__v1 - ) - - accounts_frontend_derived__login_funnels_by_service__v1.set_upstream( - wait_for_accounts_backend_derived__events_stream__v1 + accounts_frontend_derived__email_first_reg_login_funnels_by_service__v1.set_upstream( + wait_for_copy_deduplicate_all ) - accounts_frontend_derived__login_funnels_by_service__v1.set_upstream( + accounts_frontend_derived__login_engagement_funnel__v1.set_upstream( wait_for_accounts_frontend_derived__events_stream__v1 ) @@ -336,11 +335,7 @@ ) accounts_frontend_derived__registration_funnels_by_service__v1.set_upstream( - wait_for_accounts_backend_derived__events_stream__v1 - ) - - accounts_frontend_derived__registration_funnels_by_service__v1.set_upstream( - wait_for_accounts_frontend_derived__events_stream__v1 + wait_for_copy_deduplicate_all ) fenix_derived__android_onboarding__v1.set_upstream( diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-08-02 21:40:12.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-08-02 21:38:03.000000000 +0000 @@ -386,17 +386,6 @@ parameters=["conversion_window:INT64:30"], ) - mozilla_org_derived__www_site_downloads__v2 = bigquery_etl_query( - task_id="mozilla_org_derived__www_site_downloads__v2", - destination_table="www_site_downloads_v2", - dataset_id="mozilla_org_derived", - project_id="moz-fx-data-shared-prod", - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - ) - mozilla_vpn_derived__site_metrics_summary__v2 = bigquery_etl_query( task_id="mozilla_vpn_derived__site_metrics_summary__v2", destination_table="site_metrics_summary_v2", @@ -484,8 +473,6 @@ wait_for_telemetry_derived__clients_daily__v6 ) - mozilla_org_derived__www_site_downloads__v2.set_upstream(wait_for_wmo_events_table) - mozilla_vpn_derived__site_metrics_summary__v2.set_upstream( wait_for_wmo_events_table ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_mobile_kpi_metrics.py /tmp/workspace/generated-sql/dags/bqetl_mobile_kpi_metrics.py --- /tmp/workspace/main-generated-sql/dags/bqetl_mobile_kpi_metrics.py 2024-08-02 21:40:12.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_mobile_kpi_metrics.py 2024-08-02 21:38:04.000000000 +0000 @@ -62,78 +62,6 @@ task_group_klar_ios = TaskGroup("klar_ios") - wait_for_copy_deduplicate_all = ExternalTaskSensor( - task_id="wait_for_copy_deduplicate_all", - external_dag_id="copy_deduplicate", - external_task_id="copy_deduplicate_all", - execution_delta=datetime.timedelta(seconds=39600), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fenix_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.org_mozilla_firefox_derived__baseline_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=36000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 = ( ExternalTaskSensor( task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", @@ -208,10 +136,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", + wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1 = ExternalTaskSensor( + task_id="wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1", external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", + external_task_id="fenix.org_mozilla_fenix_derived__baseline_clients_daily__v1", execution_delta=datetime.timedelta(seconds=36000), check_existence=True, mode="reschedule", @@ -220,10 +148,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", + wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1 = ExternalTaskSensor( + task_id="wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", + external_task_id="fenix.org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1", execution_delta=datetime.timedelta(seconds=36000), check_existence=True, mode="reschedule", @@ -232,10 +160,34 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", + wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1 = ExternalTaskSensor( + task_id="wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", + external_task_id="fenix.org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( + task_id="wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="fenix.org_mozilla_firefox_beta_derived__baseline_clients_daily__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1 = ExternalTaskSensor( + task_id="wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="fenix.org_mozilla_firefox_derived__baseline_clients_daily__v1", execution_delta=datetime.timedelta(seconds=36000), check_existence=True, mode="reschedule", @@ -292,10 +244,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1", + wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1 = ExternalTaskSensor( + task_id="wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_beta_derived__baseline_clients_daily__v1", + external_task_id="firefox_ios.org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", execution_delta=datetime.timedelta(seconds=36000), check_existence=True, mode="reschedule", @@ -304,10 +256,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1", + wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 = ExternalTaskSensor( + task_id="wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_derived__baseline_clients_daily__v1", + external_task_id="firefox_ios.org_mozilla_ios_firefox_derived__baseline_clients_daily__v1", execution_delta=datetime.timedelta(seconds=36000), check_existence=True, mode="reschedule", @@ -316,10 +268,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", + wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( + task_id="wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", + external_task_id="firefox_ios.org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1", execution_delta=datetime.timedelta(seconds=36000), check_existence=True, mode="reschedule", @@ -364,10 +316,34 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1", + wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1 = ExternalTaskSensor( + task_id="wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1", external_dag_id="bqetl_glean_usage", - external_task_id="focus_ios.org_mozilla_ios_focus_derived__baseline_clients_daily__v1", + external_task_id="focus_android.org_mozilla_focus_beta_derived__baseline_clients_daily__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1 = ExternalTaskSensor( + task_id="wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="focus_android.org_mozilla_focus_derived__baseline_clients_daily__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1 = ExternalTaskSensor( + task_id="wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="focus_android.org_mozilla_focus_nightly_derived__baseline_clients_daily__v1", execution_delta=datetime.timedelta(seconds=36000), check_existence=True, mode="reschedule", @@ -388,10 +364,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1", + wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1 = ExternalTaskSensor( + task_id="wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1", external_dag_id="bqetl_glean_usage", - external_task_id="klar_android.org_mozilla_klar_derived__baseline_clients_daily__v1", + external_task_id="focus_ios.org_mozilla_ios_focus_derived__baseline_clients_daily__v1", execution_delta=datetime.timedelta(seconds=36000), check_existence=True, mode="reschedule", @@ -412,10 +388,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1", + wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1 = ExternalTaskSensor( + task_id="wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1", external_dag_id="bqetl_glean_usage", - external_task_id="klar_ios.org_mozilla_ios_klar_derived__baseline_clients_daily__v1", + external_task_id="klar_android.org_mozilla_klar_derived__baseline_clients_daily__v1", execution_delta=datetime.timedelta(seconds=36000), check_existence=True, mode="reschedule", @@ -436,18 +412,16 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - checks__warn_fenix_derived__attribution_clients__v1 = bigquery_dq_check( - task_id="checks__warn_fenix_derived__attribution_clients__v1", - source_table="attribution_clients_v1", - dataset_id="fenix_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - depends_on_past=False, - parameters=["submission_date:DATE:{{ds}}"], - retries=0, - task_group=task_group_fenix, + wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1 = ExternalTaskSensor( + task_id="wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1", + external_dag_id="bqetl_glean_usage", + external_task_id="klar_ios.org_mozilla_ios_klar_derived__baseline_clients_daily__v1", + execution_delta=datetime.timedelta(seconds=36000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", ) checks__warn_fenix_derived__engagement__v1 = bigquery_dq_check( @@ -479,20 +453,6 @@ task_group=task_group_fenix, ) - checks__warn_firefox_ios_derived__attribution_clients__v1 = bigquery_dq_check( - task_id="checks__warn_firefox_ios_derived__attribution_clients__v1", - source_table="attribution_clients_v1", - dataset_id="firefox_ios_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - depends_on_past=False, - parameters=["submission_date:DATE:{{ds}}"], - retries=0, - task_group=task_group_firefox_ios, - ) - checks__warn_firefox_ios_derived__engagement__v1 = bigquery_dq_check( task_id="checks__warn_firefox_ios_derived__engagement__v1", source_table="engagement_v1", @@ -522,20 +482,6 @@ task_group=task_group_firefox_ios, ) - checks__warn_focus_android_derived__attribution_clients__v1 = bigquery_dq_check( - task_id="checks__warn_focus_android_derived__attribution_clients__v1", - source_table="attribution_clients_v1", - dataset_id="focus_android_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - depends_on_past=False, - parameters=["submission_date:DATE:{{ds}}"], - retries=0, - task_group=task_group_focus_android, - ) - checks__warn_focus_android_derived__engagement__v1 = bigquery_dq_check( task_id="checks__warn_focus_android_derived__engagement__v1", source_table="engagement_v1", @@ -565,20 +511,6 @@ task_group=task_group_focus_android, ) - checks__warn_focus_ios_derived__attribution_clients__v1 = bigquery_dq_check( - task_id="checks__warn_focus_ios_derived__attribution_clients__v1", - source_table="attribution_clients_v1", - dataset_id="focus_ios_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - depends_on_past=False, - parameters=["submission_date:DATE:{{ds}}"], - retries=0, - task_group=task_group_focus_ios, - ) - checks__warn_focus_ios_derived__engagement__v1 = bigquery_dq_check( task_id="checks__warn_focus_ios_derived__engagement__v1", source_table="engagement_v1", @@ -608,20 +540,6 @@ task_group=task_group_focus_ios, ) - checks__warn_klar_android_derived__attribution_clients__v1 = bigquery_dq_check( - task_id="checks__warn_klar_android_derived__attribution_clients__v1", - source_table="attribution_clients_v1", - dataset_id="klar_android_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - depends_on_past=False, - parameters=["submission_date:DATE:{{ds}}"], - retries=0, - task_group=task_group_klar_android, - ) - checks__warn_klar_android_derived__engagement__v1 = bigquery_dq_check( task_id="checks__warn_klar_android_derived__engagement__v1", source_table="engagement_v1", @@ -651,20 +569,6 @@ task_group=task_group_klar_android, ) - checks__warn_klar_ios_derived__attribution_clients__v1 = bigquery_dq_check( - task_id="checks__warn_klar_ios_derived__attribution_clients__v1", - source_table="attribution_clients_v1", - dataset_id="klar_ios_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - depends_on_past=False, - parameters=["submission_date:DATE:{{ds}}"], - retries=0, - task_group=task_group_klar_ios, - ) - checks__warn_klar_ios_derived__engagement__v1 = bigquery_dq_check( task_id="checks__warn_klar_ios_derived__engagement__v1", source_table="engagement_v1", @@ -694,18 +598,6 @@ task_group=task_group_klar_ios, ) - fenix_derived__attribution_clients__v1 = bigquery_etl_query( - task_id="fenix_derived__attribution_clients__v1", - destination_table="attribution_clients_v1", - dataset_id="fenix_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_fenix, - ) - fenix_derived__engagement__v1 = bigquery_etl_query( task_id="fenix_derived__engagement__v1", destination_table="engagement_v1", @@ -732,18 +624,6 @@ task_group=task_group_fenix, ) - firefox_ios_derived__attribution_clients__v1 = bigquery_etl_query( - task_id="firefox_ios_derived__attribution_clients__v1", - destination_table="attribution_clients_v1", - dataset_id="firefox_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_firefox_ios, - ) - firefox_ios_derived__engagement__v1 = bigquery_etl_query( task_id="firefox_ios_derived__engagement__v1", destination_table="engagement_v1", @@ -770,18 +650,6 @@ task_group=task_group_firefox_ios, ) - focus_android_derived__attribution_clients__v1 = bigquery_etl_query( - task_id="focus_android_derived__attribution_clients__v1", - destination_table="attribution_clients_v1", - dataset_id="focus_android_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_focus_android, - ) - focus_android_derived__engagement__v1 = bigquery_etl_query( task_id="focus_android_derived__engagement__v1", destination_table="engagement_v1", @@ -808,18 +676,6 @@ task_group=task_group_focus_android, ) - focus_ios_derived__attribution_clients__v1 = bigquery_etl_query( - task_id="focus_ios_derived__attribution_clients__v1", - destination_table="attribution_clients_v1", - dataset_id="focus_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_focus_ios, - ) - focus_ios_derived__engagement__v1 = bigquery_etl_query( task_id="focus_ios_derived__engagement__v1", destination_table="engagement_v1", @@ -846,18 +702,6 @@ task_group=task_group_focus_ios, ) - klar_android_derived__attribution_clients__v1 = bigquery_etl_query( - task_id="klar_android_derived__attribution_clients__v1", - destination_table="attribution_clients_v1", - dataset_id="klar_android_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_klar_android, - ) - klar_android_derived__engagement__v1 = bigquery_etl_query( task_id="klar_android_derived__engagement__v1", destination_table="engagement_v1", @@ -884,18 +728,6 @@ task_group=task_group_klar_android, ) - klar_ios_derived__attribution_clients__v1 = bigquery_etl_query( - task_id="klar_ios_derived__attribution_clients__v1", - destination_table="attribution_clients_v1", - dataset_id="klar_ios_derived", - project_id="moz-fx-data-shared-prod", - owner="mozilla/kpi_table_reviewers", - email=["kik@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter="submission_date", - depends_on_past=False, - task_group=task_group_klar_ios, - ) - klar_ios_derived__engagement__v1 = bigquery_etl_query( task_id="klar_ios_derived__engagement__v1", destination_table="engagement_v1", @@ -922,20 +754,12 @@ task_group=task_group_klar_ios, ) - checks__warn_fenix_derived__attribution_clients__v1.set_upstream( - fenix_derived__attribution_clients__v1 - ) - checks__warn_fenix_derived__engagement__v1.set_upstream( fenix_derived__engagement__v1 ) checks__warn_fenix_derived__retention__v1.set_upstream(fenix_derived__retention__v1) - checks__warn_firefox_ios_derived__attribution_clients__v1.set_upstream( - firefox_ios_derived__attribution_clients__v1 - ) - checks__warn_firefox_ios_derived__engagement__v1.set_upstream( firefox_ios_derived__engagement__v1 ) @@ -944,10 +768,6 @@ firefox_ios_derived__retention__v1 ) - checks__warn_focus_android_derived__attribution_clients__v1.set_upstream( - focus_android_derived__attribution_clients__v1 - ) - checks__warn_focus_android_derived__engagement__v1.set_upstream( focus_android_derived__engagement__v1 ) @@ -956,10 +776,6 @@ focus_android_derived__retention__v1 ) - checks__warn_focus_ios_derived__attribution_clients__v1.set_upstream( - focus_ios_derived__attribution_clients__v1 - ) - checks__warn_focus_ios_derived__engagement__v1.set_upstream( focus_ios_derived__engagement__v1 ) @@ -968,10 +784,6 @@ focus_ios_derived__retention__v1 ) - checks__warn_klar_android_derived__attribution_clients__v1.set_upstream( - klar_android_derived__attribution_clients__v1 - ) - checks__warn_klar_android_derived__engagement__v1.set_upstream( klar_android_derived__engagement__v1 ) @@ -980,10 +792,6 @@ klar_android_derived__retention__v1 ) - checks__warn_klar_ios_derived__attribution_clients__v1.set_upstream( - klar_ios_derived__attribution_clients__v1 - ) - checks__warn_klar_ios_derived__engagement__v1.set_upstream( klar_ios_derived__engagement__v1 ) @@ -992,28 +800,6 @@ klar_ios_derived__retention__v1 ) - fenix_derived__attribution_clients__v1.set_upstream(wait_for_copy_deduplicate_all) - - fenix_derived__attribution_clients__v1.set_upstream( - wait_for_org_mozilla_fenix_derived__baseline_clients_daily__v1 - ) - - fenix_derived__attribution_clients__v1.set_upstream( - wait_for_org_mozilla_fenix_nightly_derived__baseline_clients_daily__v1 - ) - - fenix_derived__attribution_clients__v1.set_upstream( - wait_for_org_mozilla_fennec_aurora_derived__baseline_clients_daily__v1 - ) - - fenix_derived__attribution_clients__v1.set_upstream( - wait_for_org_mozilla_firefox_beta_derived__baseline_clients_daily__v1 - ) - - fenix_derived__attribution_clients__v1.set_upstream( - wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1 - ) - fenix_derived__engagement__v1.set_upstream( wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 ) @@ -1082,22 +868,6 @@ wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1 ) - firefox_ios_derived__attribution_clients__v1.set_upstream( - wait_for_copy_deduplicate_all - ) - - firefox_ios_derived__attribution_clients__v1.set_upstream( - wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1 - ) - - firefox_ios_derived__attribution_clients__v1.set_upstream( - wait_for_org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 - ) - - firefox_ios_derived__attribution_clients__v1.set_upstream( - wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 - ) - firefox_ios_derived__engagement__v1.set_upstream( wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 ) @@ -1142,18 +912,6 @@ wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 ) - focus_android_derived__attribution_clients__v1.set_upstream( - wait_for_org_mozilla_focus_beta_derived__baseline_clients_daily__v1 - ) - - focus_android_derived__attribution_clients__v1.set_upstream( - wait_for_org_mozilla_focus_derived__baseline_clients_daily__v1 - ) - - focus_android_derived__attribution_clients__v1.set_upstream( - wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1 - ) - focus_android_derived__engagement__v1.set_upstream( wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 ) @@ -1190,10 +948,6 @@ wait_for_org_mozilla_focus_nightly_derived__baseline_clients_daily__v1 ) - focus_ios_derived__attribution_clients__v1.set_upstream( - wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1 - ) - focus_ios_derived__engagement__v1.set_upstream( wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 ) @@ -1206,10 +960,6 @@ wait_for_org_mozilla_ios_focus_derived__baseline_clients_daily__v1 ) - klar_android_derived__attribution_clients__v1.set_upstream( - wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1 - ) - klar_android_derived__engagement__v1.set_upstream( wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1 ) @@ -1222,10 +972,6 @@ wait_for_org_mozilla_klar_derived__baseline_clients_daily__v1 ) - klar_ios_derived__attribution_clients__v1.set_upstream( - wait_for_org_mozilla_ios_klar_derived__baseline_clients_daily__v1 - ) - klar_ios_derived__engagement__v1.set_upstream( wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_monitoring.py /tmp/workspace/generated-sql/dags/bqetl_monitoring.py --- /tmp/workspace/main-generated-sql/dags/bqetl_monitoring.py 2024-08-02 21:40:12.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_monitoring.py 2024-08-02 21:38:01.000000000 +0000 @@ -295,17 +295,6 @@ email=["amiyaguchi@mozilla.com", "ascholtz@mozilla.com"], ) - monitoring_derived__table_partition_expirations__v1 = bigquery_etl_query( - task_id="monitoring_derived__table_partition_expirations__v1", - destination_table="table_partition_expirations_v1", - dataset_id="monitoring_derived", - project_id="moz-fx-data-shared-prod", - owner="bewu@mozilla.cam", - email=["ascholtz@mozilla.com", "bewu@mozilla.cam"], - date_partition_parameter="submission_date", - depends_on_past=True, - ) - monitoring_derived__telemetry_missing_columns__v3 = bigquery_etl_query( task_id="monitoring_derived__telemetry_missing_columns__v3", destination_table="telemetry_missing_columns_v3", @@ -361,10 +350,6 @@ wait_for_copy_deduplicate_all ) - monitoring_derived__table_partition_expirations__v1.set_upstream( - wait_for_copy_deduplicate_all - ) - monitoring_derived__telemetry_missing_columns__v3.set_upstream( wait_for_copy_deduplicate_all ) Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external: fxa_account_groups_v1 Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external: fxa_carts_v1 Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external: fxa_groups_v1 Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external: fxa_account_groups_v1 Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external: fxa_carts_v1 Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_nonprod_external: fxa_groups_v1 Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_frontend_derived: email_first_reg_login_funnels_v1 Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/ads/ppa_measurements: schema.yaml Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_errors_v1: checks.sql Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/browser_usage_v1: checks.sql Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_errors_v1: checks.sql Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/device_usage_v1: checks.sql Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_errors_v1: checks.sql Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1: checks.sql Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix: attribution_clients Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived: attribution_clients_v1 Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios: attribution_clients Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived: attribution_clients_v1 Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/focus_android: attribution_clients Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/focus_android_derived: attribution_clients_v1 Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/focus_ios: attribution_clients Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/focus_ios_derived: attribution_clients_v1 Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_android: attribution_clients Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived: attribution_clients_v1 Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_ios: attribution_clients Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_ios_derived: attribution_clients_v1 Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress: schema.yaml Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived: table_partition_expirations_v1 Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: www_site_downloads Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: www_site_downloads_v2 Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod: phabricator_metrics Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/pocket/pocket_reach_mau: schema.yaml Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_terms/aggregated_search_terms_daily: schema.yaml Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_terms/sanitization_job_data_validation_metrics: schema.yaml Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_terms/sanitization_job_languages: schema.yaml Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_terms/search_terms_daily: schema.yaml Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/buildhub2: schema.yaml Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry: mobile_attribution_clients Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived: glam_client_probe_counts_extract_v1 Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/glam_extract_firefox_beta_v1: query.sql Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/glam_extract_firefox_beta_v1: schema.yaml Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/glam_extract_firefox_nightly_v1: query.sql Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/glam_extract_firefox_nightly_v1: schema.yaml Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/glam_extract_firefox_release_v1: query.sql Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/glam_extract_firefox_release_v1: schema.yaml diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_daily_v1/checks.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_daily_v1/checks.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_daily_v1/checks.sql 2024-08-02 21:34:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_backend_derived/users_services_daily_v1/checks.sql 2024-08-02 21:16:45.000000000 +0000 @@ -110,8 +110,6 @@ USING (day, event_name) WHERE event_name IS NOT NULL - -- glean_page_load events are automatically sent only in `events` ping - AND event_name != 'glean_page_load' -- fix in progress in https://github.com/mozilla/fxa/pull/17218 -- will be removed from here when this lands in production AND event_name NOT IN ( @@ -148,7 +146,7 @@ OR ABS(events_new.count_new - events_old.count_old) / LEAST( events_new.count_new, events_old.count_old - ) > 0.1 -- low-volume events can have higher relative discrepancies + ) > 0.05 ) ) SELECT diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_groups_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_groups_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_groups_v1/metadata.yaml 2024-08-02 21:35:50.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_groups_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 @@ -1,22 +0,0 @@ -friendly_name: accountGroups table from production fxa database -description: | - A mirror of the `accountGroups` table from the production `fxa` CloudSQL database, excluding columns containing confidential data, updated daily to match the current state of the table. - See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa -owners: -- akomar@mozilla.com -labels: - application: accounts_backend - schedule: daily - dag: bqetl_accounts_backend_external - owner1: akomar -scheduling: - dag_name: bqetl_accounts_backend_external - date_partition_parameter: null -bigquery: null -workgroup_access: -- role: roles/bigquery.dataViewer - members: - - workgroup:accounts-confidential -references: - query.sql: - - EXTERNAL_QUERY('moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa', 'SELECT\n uid,\n group_id,\n role,\n managed_by,\n expires,\n notes\n FROM\n fxa.accountGroups\n ') diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_groups_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_groups_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_groups_v1/query.sql 2024-08-02 21:34:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_groups_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 @@ -1,21 +0,0 @@ -SELECT - TO_HEX(uid) AS uid, - group_id, - role, - managed_by, - SAFE.TIMESTAMP_MILLIS(SAFE_CAST(expires AS INT)) AS expires, - notes, -FROM - EXTERNAL_QUERY( - "moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa", - """SELECT - uid, - group_id, - role, - managed_by, - expires, - notes - FROM - fxa.accountGroups - """ - ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_groups_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_groups_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_groups_v1/schema.yaml 2024-08-02 21:34:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_account_groups_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 @@ -1,20 +0,0 @@ -fields: -- name: uid - type: STRING - mode: NULLABLE - description: Account ID in hexadecimal format. -- name: group_id - type: INTEGER - mode: NULLABLE -- name: role - type: STRING - mode: NULLABLE -- name: managed_by - type: STRING - mode: NULLABLE -- name: expires - type: TIMESTAMP - mode: NULLABLE -- name: notes - type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_accounts_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_accounts_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_accounts_v1/metadata.yaml 2024-08-02 21:35:50.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_accounts_v1/metadata.yaml 2024-08-02 21:32:39.000000000 +0000 @@ -19,4 +19,4 @@ - workgroup:accounts-confidential references: query.sql: - - EXTERNAL_QUERY('moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa', 'SELECT\n uid,\n normalizedEmail,\n email,\n emailVerified,\n verifierVersion,\n verifierSetAt,\n createdAt,\n locale,\n lockedAt,\n profileChangedAt,\n keysChangedAt,\n ecosystemAnonId,\n disabledAt,\n metricsOptOutAt,\n atLeast18AtReg\n FROM\n fxa.accounts\n ') + - EXTERNAL_QUERY('moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa', 'SELECT\n uid,\n normalizedEmail,\n email,\n emailVerified,\n verifierVersion,\n verifierSetAt,\n createdAt,\n locale,\n lockedAt,\n profileChangedAt,\n keysChangedAt,\n ecosystemAnonId,\n disabledAt,\n metricsOptOutAt\n FROM\n fxa.accounts\n ') diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_accounts_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_accounts_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_accounts_v1/query.sql 2024-08-02 21:34:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_accounts_v1/query.sql 2024-08-02 21:16:45.000000000 +0000 @@ -13,7 +13,6 @@ ecosystemAnonId, SAFE.TIMESTAMP_MILLIS(SAFE_CAST(disabledAt AS INT)) AS disabledAt, SAFE.TIMESTAMP_MILLIS(SAFE_CAST(metricsOptOutAt AS INT)) AS metricsOptOutAt, - SAFE_CAST(atLeast18AtReg AS BOOL) AS atLeast18AtReg, FROM EXTERNAL_QUERY( "moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa", @@ -31,8 +30,7 @@ keysChangedAt, ecosystemAnonId, disabledAt, - metricsOptOutAt, - atLeast18AtReg + metricsOptOutAt FROM fxa.accounts """ diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_accounts_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_accounts_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_accounts_v1/schema.yaml 2024-08-02 21:34:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_accounts_v1/schema.yaml 2024-08-02 21:16:45.000000000 +0000 @@ -42,6 +42,3 @@ - name: metricsOptOutAt type: TIMESTAMP mode: NULLABLE -- name: atLeast18AtReg - type: BOOLEAN - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_carts_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_carts_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_carts_v1/metadata.yaml 2024-08-02 21:35:50.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_carts_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 @@ -1,22 +0,0 @@ -friendly_name: carts table from production fxa database -description: | - A mirror of the `carts` table from the production `fxa` CloudSQL database, excluding columns containing confidential data, updated daily to match the current state of the table. - See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa -owners: -- akomar@mozilla.com -labels: - application: accounts_backend - schedule: daily - dag: bqetl_accounts_backend_external - owner1: akomar -scheduling: - dag_name: bqetl_accounts_backend_external - date_partition_parameter: null -bigquery: null -workgroup_access: -- role: roles/bigquery.dataViewer - members: - - workgroup:accounts-confidential -references: - query.sql: - - EXTERNAL_QUERY('moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa', 'SELECT\n id,\n uid,\n state,\n errorReasonId,\n offeringConfigId,\n interval,\n experiment,\n taxAddress,\n createdAt,\n updatedAt,\n couponCode,\n stripeCustomerId,\n email,\n amount,\n version,\n eligibilityStatus\n FROM\n fxa.carts\n ') diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_carts_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_carts_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_carts_v1/query.sql 2024-08-02 21:34:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_carts_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 @@ -1,41 +0,0 @@ -SELECT - TO_HEX(id) AS id, - TO_HEX(uid) AS uid, - state, - errorReasonId, - offeringConfigId, - `interval`, - experiment, - taxAddress, - SAFE.TIMESTAMP_MILLIS(SAFE_CAST(createdAt AS INT)) AS createdAt, - SAFE.TIMESTAMP_MILLIS(SAFE_CAST(updatedAt AS INT)) AS updatedAt, - couponCode, - stripeCustomerId, - email, - amount, - version, - eligibilityStatus, -FROM - EXTERNAL_QUERY( - "moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa", - """SELECT - id, - uid, - state, - errorReasonId, - offeringConfigId, - `interval`, - experiment, - taxAddress, - createdAt, - updatedAt, - couponCode, - stripeCustomerId, - email, - amount, - version, - eligibilityStatus - FROM - fxa.carts - """ - ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_carts_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_carts_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_carts_v1/schema.yaml 2024-08-02 21:34:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_carts_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 @@ -1,51 +0,0 @@ -fields: -- name: id - type: STRING - mode: NULLABLE - description: Cart ID in hexadecimal format. -- name: uid - type: STRING - mode: NULLABLE - description: Account ID in hexadecimal format. -- name: state - type: STRING - mode: NULLABLE -- name: errorReasonId - type: STRING - mode: NULLABLE -- name: offeringConfigId - type: STRING - mode: NULLABLE -- name: interval - type: STRING - mode: NULLABLE -- name: experiment - type: STRING - mode: NULLABLE -- name: taxAddress - type: STRING - mode: NULLABLE -- name: createdAt - type: TIMESTAMP - mode: NULLABLE -- name: updatedAt - type: TIMESTAMP - mode: NULLABLE -- name: couponCode - type: STRING - mode: NULLABLE -- name: stripeCustomerId - type: STRING - mode: NULLABLE -- name: email - type: STRING - mode: NULLABLE -- name: amount - type: INTEGER - mode: NULLABLE -- name: version - type: INTEGER - mode: NULLABLE -- name: eligibilityStatus - type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_groups_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_groups_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_groups_v1/metadata.yaml 2024-08-02 21:35:50.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_groups_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 @@ -1,22 +0,0 @@ -friendly_name: groups table from production fxa database -description: | - A mirror of the `groups` table from the production `fxa` CloudSQL database, excluding columns containing confidential data, updated daily to match the current state of the table. - See https://mozilla.github.io/ecosystem-platform/reference/database-structure#database-fxa -owners: -- akomar@mozilla.com -labels: - application: accounts_backend - schedule: daily - dag: bqetl_accounts_backend_external - owner1: akomar -scheduling: - dag_name: bqetl_accounts_backend_external - date_partition_parameter: null -bigquery: null -workgroup_access: -- role: roles/bigquery.dataViewer - members: - - workgroup:accounts-confidential -references: - query.sql: - - EXTERNAL_QUERY('moz-fx-fxa-prod.us.fxa-rds-prod-prod-fxa', 'SELECT\n id,\n name,\n display_name,\n capabilities\n FROM\n fxa.groups\n ') diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_groups_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_groups_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_db_external/fxa_groups_v1/query.sql 2024-08-02 21:34:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql ```⚠️ Only part of the diff is displayed.
corbanCloud
Checklist for reviewer:
<username>:<branch>of the fork as parameter. The parameter will also show up in the logs of themanual-trigger-required-for-forkCI task together with more detailed instructions.For modifications to schemas in restricted namespaces (see
CODEOWNERS):┆Issue is synchronized with this Jira Task