dataops-ci-bot
commented
1 month ago Integration report for "Update mobile_search_clients_daily tests"
sql.diff
Click to expand!
```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_analytics_aggregations.py /tmp/workspace/generated-sql/dags/bqetl_analytics_aggregations.py --- /tmp/workspace/main-generated-sql/dags/bqetl_analytics_aggregations.py 2024-05-31 17:20:21.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_analytics_aggregations.py 2024-05-31 17:31:47.000000000 +0000 @@ -190,18 +190,6 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_search_derived__mobile_search_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_mobile_search", - external_task_id="search_derived__mobile_search_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - wait_for_firefox_ios_derived__clients_last_seen_joined__v1 = ExternalTaskSensor( task_id="wait_for_firefox_ios_derived__clients_last_seen_joined__v1", external_dag_id="bqetl_glean_usage", @@ -860,10 +848,6 @@ wait_for_fenix_derived__clients_last_seen_joined__v1 ) - fenix_active_users_aggregates.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) - firefox_desktop_active_users_aggregates.set_upstream( wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 ) @@ -884,19 +868,11 @@ wait_for_firefox_ios_derived__clients_last_seen_joined__v1 ) - firefox_ios_active_users_aggregates.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) - focus_android_active_users_aggregates.set_upstream( wait_for_focus_android_derived__clients_last_seen_joined__v1 ) focus_android_active_users_aggregates.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) - - focus_android_active_users_aggregates.set_upstream( wait_for_telemetry_derived__core_clients_last_seen__v1 ) @@ -916,10 +892,6 @@ wait_for_focus_ios_derived__clients_last_seen_joined__v1 ) - focus_ios_active_users_aggregates.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) - klar_ios_active_users_aggregates.set_upstream( wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 ) @@ -936,10 +908,6 @@ wait_for_klar_ios_derived__clients_last_seen_joined__v1 ) - klar_ios_active_users_aggregates.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) - telemetry_derived__cohort_daily_statistics__v1.set_upstream( wait_for_checks__fail_telemetry_derived__unified_metrics__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_desktop_mobile_search_monthly.py /tmp/workspace/generated-sql/dags/bqetl_desktop_mobile_search_monthly.py --- /tmp/workspace/main-generated-sql/dags/bqetl_desktop_mobile_search_monthly.py 2024-05-31 17:20:21.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_desktop_mobile_search_monthly.py 2024-05-31 17:31:46.000000000 +0000 @@ -50,18 +50,6 @@ tags=tags, ) as dag: - wait_for_search_derived__mobile_search_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_mobile_search", - external_task_id="search_derived__mobile_search_clients_daily__v1", - execution_delta=datetime.timedelta(days=2, seconds=10800), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - wait_for_search_derived__search_clients_daily__v8 = ExternalTaskSensor( task_id="wait_for_search_derived__search_clients_daily__v8", external_dag_id="bqetl_search", @@ -87,9 +75,5 @@ ) search_derived__desktop_mobile_search_clients_monthly__v1.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) - - search_derived__desktop_mobile_search_clients_monthly__v1.set_upstream( wait_for_search_derived__search_clients_daily__v8 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_firefox_ios.py /tmp/workspace/generated-sql/dags/bqetl_firefox_ios.py --- /tmp/workspace/main-generated-sql/dags/bqetl_firefox_ios.py 2024-05-31 17:20:21.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_firefox_ios.py 2024-05-31 17:31:46.000000000 +0000 @@ -86,18 +86,6 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_search_derived__mobile_search_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_mobile_search", - external_task_id="search_derived__mobile_search_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=7200), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1 = ExternalTaskSensor( task_id="wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1", external_dag_id="bqetl_glean_usage", @@ -705,10 +693,6 @@ wait_for_copy_deduplicate_all ) - firefox_ios_derived__attributable_clients__v1.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) - firefox_ios_derived__baseline_clients_yearly__v1.set_upstream( wait_for_org_mozilla_ios_fennec_derived__baseline_clients_daily__v1 ) @@ -741,10 +725,6 @@ wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 ) - firefox_ios_derived__clients_activation__v1.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) - firefox_ios_derived__firefox_ios_clients__v1.set_upstream( wait_for_copy_deduplicate_all ) @@ -816,7 +796,3 @@ firefox_ios_derived__new_profile_activation__v2.set_upstream( wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 ) - - firefox_ios_derived__new_profile_activation__v2.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_mobile_activation.py /tmp/workspace/generated-sql/dags/bqetl_mobile_activation.py --- /tmp/workspace/main-generated-sql/dags/bqetl_mobile_activation.py 2024-05-31 17:20:21.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_mobile_activation.py 2024-05-31 17:31:47.000000000 +0000 @@ -182,18 +182,6 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_search_derived__mobile_search_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_mobile_search", - external_task_id="search_derived__mobile_search_clients_daily__v1", - execution_delta=datetime.timedelta(days=-1, seconds=79200), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", external_dag_id="bqetl_glean_usage", @@ -353,10 +341,6 @@ wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1 ) - fenix_derived__new_profile_activation__v1.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) - firefox_ios_derived__new_profile_activation__v1.set_upstream( wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 ) @@ -380,7 +364,3 @@ firefox_ios_derived__new_profile_activation__v1.set_upstream( wait_for_org_mozilla_ios_firefoxbeta_derived__baseline_clients_daily__v1 ) - - firefox_ios_derived__new_profile_activation__v1.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_mobile_search.py /tmp/workspace/generated-sql/dags/bqetl_mobile_search.py --- /tmp/workspace/main-generated-sql/dags/bqetl_mobile_search.py 2024-05-31 17:20:21.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_mobile_search.py 2024-05-31 17:31:45.000000000 +0000 @@ -52,18 +52,6 @@ tags=tags, ) as dag: - wait_for_copy_deduplicate_all = ExternalTaskSensor( - task_id="wait_for_copy_deduplicate_all", - external_dag_id="copy_deduplicate", - external_task_id="copy_deduplicate_all", - execution_delta=datetime.timedelta(seconds=3600), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - search_derived__mobile_search_aggregates__v1 = bigquery_etl_query( task_id="search_derived__mobile_search_aggregates__v1", destination_table="mobile_search_aggregates_v1", @@ -80,90 +68,6 @@ depends_on_past=False, ) - search_derived__mobile_search_clients_daily__v1 = bigquery_etl_query( - task_id="search_derived__mobile_search_clients_daily__v1", - destination_table="mobile_search_clients_daily_v1", - dataset_id="search_derived", - project_id="moz-fx-data-shared-prod", - owner="akomar@mozilla.com", - email=[ - "akomar@mozilla.com", - "anicholson@mozilla.com", - "cmorales@mozilla.com", - "telemetry-alerts@mozilla.com", - ], - date_partition_parameter="submission_date", - depends_on_past=False, - ) - - with TaskGroup( - "search_derived__mobile_search_clients_daily__v1_external", - ) as search_derived__mobile_search_clients_daily__v1_external: - ExternalTaskMarker( - task_id="bqetl_analytics_aggregations__wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_analytics_aggregations", - external_task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=78300)).isoformat() }}", - ) - - ExternalTaskMarker( - task_id="bqetl_kpis_shredder__wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_kpis_shredder", - external_task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - ) - - ExternalTaskMarker( - task_id="bqetl_org_mozilla_firefox_derived__wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_org_mozilla_firefox_derived", - external_task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - ) - - ExternalTaskMarker( - task_id="bqetl_mobile_activation__wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_mobile_activation", - external_task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", - ) - - ExternalTaskMarker( - task_id="bqetl_firefox_ios__wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_firefox_ios", - external_task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=79200)).isoformat() }}", - ) - - ExternalTaskMarker( - task_id="bqetl_review_checker__wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_review_checker", - external_task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(seconds=7200)).isoformat() }}", - ) - - ExternalTaskMarker( - task_id="bqetl_desktop_mobile_search_monthly__wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_desktop_mobile_search_monthly", - external_task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-3, seconds=75600)).isoformat() }}", - ) - - ExternalTaskMarker( - task_id="bqetl_search_dashboard__wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_search_dashboard", - external_task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=77400)).isoformat() }}", - ) - - ExternalTaskMarker( - task_id="bqetl_unified__wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_unified", - external_task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=82800)).isoformat() }}", - ) - - search_derived__mobile_search_clients_daily__v1_external.set_upstream( - search_derived__mobile_search_clients_daily__v1 - ) - search_derived__mobile_search_clients_last_seen__v1 = bigquery_etl_query( task_id="search_derived__mobile_search_clients_last_seen__v1", destination_table="mobile_search_clients_last_seen_v1", @@ -179,15 +83,3 @@ date_partition_parameter="submission_date", depends_on_past=True, ) - - search_derived__mobile_search_aggregates__v1.set_upstream( - search_derived__mobile_search_clients_daily__v1 - ) - - search_derived__mobile_search_clients_daily__v1.set_upstream( - wait_for_copy_deduplicate_all - ) - - search_derived__mobile_search_clients_last_seen__v1.set_upstream( - search_derived__mobile_search_clients_daily__v1 - ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_org_mozilla_firefox_derived.py /tmp/workspace/generated-sql/dags/bqetl_org_mozilla_firefox_derived.py --- /tmp/workspace/main-generated-sql/dags/bqetl_org_mozilla_firefox_derived.py 2024-05-31 17:20:21.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_org_mozilla_firefox_derived.py 2024-05-31 17:31:45.000000000 +0000 @@ -139,17 +139,6 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_search_derived__mobile_search_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_mobile_search", - external_task_id="search_derived__mobile_search_clients_daily__v1", - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - wait_for_checks__fail_firefox_ios_derived__baseline_clients_yearly__v1 = ExternalTaskSensor( task_id="wait_for_checks__fail_firefox_ios_derived__baseline_clients_yearly__v1", external_dag_id="bqetl_firefox_ios", @@ -531,20 +520,12 @@ wait_for_org_mozilla_firefox_derived__baseline_clients_daily__v1 ) - fenix_derived__attributable_clients__v1.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) - fenix_derived__attributable_clients__v2.set_upstream(wait_for_copy_deduplicate_all) fenix_derived__attributable_clients__v2.set_upstream( wait_for_fenix_derived__new_profile_activation__v1 ) - fenix_derived__attributable_clients__v2.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) - fenix_derived__client_adclicks_history__v1.set_upstream( fenix_derived__attributable_clients__v2 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_review_checker.py /tmp/workspace/generated-sql/dags/bqetl_review_checker.py --- /tmp/workspace/main-generated-sql/dags/bqetl_review_checker.py 2024-05-31 17:20:21.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_review_checker.py 2024-05-31 17:31:47.000000000 +0000 @@ -112,18 +112,6 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_search_derived__mobile_search_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_mobile_search", - external_task_id="search_derived__mobile_search_clients_daily__v1", - execution_delta=datetime.timedelta(days=-1, seconds=79200), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - firefox_desktop_review_checker_clients__v1 = bigquery_etl_query( task_id="firefox_desktop_review_checker_clients__v1", destination_table="review_checker_clients_v1", @@ -261,10 +249,6 @@ wait_for_copy_deduplicate_all ) - org_mozilla_fenix_review_checker_clients__v1.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) - org_mozilla_fenix_review_checker_events__v1.set_upstream( wait_for_copy_deduplicate_all ) @@ -273,10 +257,6 @@ wait_for_copy_deduplicate_all ) - org_mozilla_ios_firefox_review_checker_clients__v1.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) - org_mozilla_ios_firefox_review_checker_events__v1.set_upstream( wait_for_copy_deduplicate_all ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_search_dashboard.py /tmp/workspace/generated-sql/dags/bqetl_search_dashboard.py --- /tmp/workspace/main-generated-sql/dags/bqetl_search_dashboard.py 2024-05-31 17:20:21.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_search_dashboard.py 2024-05-31 17:31:45.000000000 +0000 @@ -71,18 +71,6 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_search_derived__mobile_search_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_mobile_search", - external_task_id="search_derived__mobile_search_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=9000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - wait_for_checks__fail_fenix_derived__active_users_aggregates__v3 = ( ExternalTaskSensor( task_id="wait_for_checks__fail_fenix_derived__active_users_aggregates__v3", @@ -241,10 +229,6 @@ wait_for_search_derived__search_aggregates__v8 ) - search_derived__mobile_search_aggregates_for_searchreport__v1.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) - search_derived__search_revenue_levers_daily__v1.set_upstream( wait_for_checks__fail_fenix_derived__active_users_aggregates__v3 ) @@ -270,9 +254,5 @@ ) search_derived__search_revenue_levers_daily__v1.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) - - search_derived__search_revenue_levers_daily__v1.set_upstream( wait_for_search_derived__search_clients_daily__v8 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_unified.py /tmp/workspace/generated-sql/dags/bqetl_unified.py --- /tmp/workspace/main-generated-sql/dags/bqetl_unified.py 2024-05-31 17:20:21.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_unified.py 2024-05-31 17:31:46.000000000 +0000 @@ -128,18 +128,6 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_search_derived__mobile_search_clients_daily__v1 = ExternalTaskSensor( - task_id="wait_for_search_derived__mobile_search_clients_daily__v1", - external_dag_id="bqetl_mobile_search", - external_task_id="search_derived__mobile_search_clients_daily__v1", - execution_delta=datetime.timedelta(seconds=3600), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - wait_for_telemetry_derived__core_clients_last_seen__v1 = ExternalTaskSensor( task_id="wait_for_telemetry_derived__core_clients_last_seen__v1", external_dag_id="bqetl_core", @@ -341,9 +329,5 @@ ) telemetry_derived__unified_metrics__v1.set_upstream( - wait_for_search_derived__mobile_search_clients_daily__v1 - ) - - telemetry_derived__unified_metrics__v1.set_upstream( wait_for_telemetry_derived__core_clients_last_seen__v1 ) Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search/mobile_search_clients_engines_sources_daily: schema.yaml diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:17:27.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:19:30.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.events_v1` + `moz-fx-data-shared-prod.bedrock_live.interaction_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-05-31 17:16:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-05-31 17:25:14.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: source + mode: NULLABLE +- name: source type: STRING -- mode: NULLABLE - name: event_type + mode: NULLABLE +- name: event_type type: STRING -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: subdivision1 + mode: NULLABLE +- name: subdivision1 type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: event_count + mode: NULLABLE +- name: event_count type: INTEGER -- mode: NULLABLE - name: user_count + mode: NULLABLE +- name: user_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-05-31 17:16:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-05-31 17:25:15.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: impression_count + mode: NULLABLE +- name: impression_count type: INTEGER -- mode: NULLABLE - name: click_count + mode: NULLABLE +- name: click_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-05-31 17:16:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-05-31 17:25:32.000000000 +0000 @@ -26,6 +26,9 @@ - name: adjust_network type: STRING mode: NULLABLE +- name: install_source + type: STRING + mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-05-31 17:16:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-05-31 17:25:32.000000000 +0000 @@ -48,6 +48,10 @@ description: 'The type of source of a client installation. ' +- name: install_source + type: STRING + mode: NULLABLE + description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:17:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:19:30.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:17:27.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:19:31.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -80,7 +80,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-05-31 17:16:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-05-31 17:24:40.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: country +- name: country type: STRING -- mode: NULLABLE - name: city + mode: NULLABLE +- name: city type: STRING -- mode: NULLABLE - name: datetime + mode: NULLABLE +- name: datetime type: TIMESTAMP -- mode: NULLABLE - name: proportion_undefined + mode: NULLABLE +- name: proportion_undefined type: FLOAT -- mode: NULLABLE - name: proportion_timeout + mode: NULLABLE +- name: proportion_timeout type: FLOAT -- mode: NULLABLE - name: proportion_abort + mode: NULLABLE +- name: proportion_abort type: FLOAT -- mode: NULLABLE - name: proportion_unreachable + mode: NULLABLE +- name: proportion_unreachable type: FLOAT -- mode: NULLABLE - name: proportion_terminated + mode: NULLABLE +- name: proportion_terminated type: FLOAT -- mode: NULLABLE - name: proportion_channel_open + mode: NULLABLE +- name: proportion_channel_open type: FLOAT -- mode: NULLABLE - name: avg_dns_success_time + mode: NULLABLE +- name: avg_dns_success_time type: FLOAT -- mode: NULLABLE - name: missing_dns_success + mode: NULLABLE +- name: missing_dns_success type: FLOAT -- mode: NULLABLE - name: avg_dns_failure_time + mode: NULLABLE +- name: avg_dns_failure_time type: FLOAT -- mode: NULLABLE - name: missing_dns_failure + mode: NULLABLE +- name: missing_dns_failure type: FLOAT -- mode: NULLABLE - name: count_dns_failure + mode: NULLABLE +- name: count_dns_failure type: FLOAT -- mode: NULLABLE - name: ssl_error_prop + mode: NULLABLE +- name: ssl_error_prop type: FLOAT -- mode: NULLABLE - name: avg_tls_handshake_time + mode: NULLABLE +- name: avg_tls_handshake_time type: FLOAT + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-05-31 17:17:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-05-31 17:21:08.000000000 +0000 @@ -45,7 +45,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -75,7 +75,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -572,7 +572,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -582,7 +582,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -663,7 +663,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -673,7 +673,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -754,7 +754,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -764,7 +764,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -1565,7 +1565,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1575,7 +1575,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1656,7 +1656,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1666,7 +1666,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1747,7 +1747,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1757,7 +1757,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1838,7 +1838,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1848,7 +1848,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2000,7 +2000,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.events_v1` + `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2010,7 +2010,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2162,7 +2162,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -2172,7 +2172,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:17:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:19:30.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.main_v1` + `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-05-31 17:16:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-05-31 17:24:42.000000000 +0000 @@ -1,7 +1,10 @@ fields: -- type: DATETIME - name: build_hour -- type: INTEGER - name: geckoview_major_version -- type: INTEGER - name: n_pings +- name: build_hour + type: DATETIME + mode: NULLABLE +- name: geckoview_major_version + type: INTEGER + mode: NULLABLE +- name: n_pings + type: INTEGER + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:17:27.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:19:32.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:17:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:19:33.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:17:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:19:33.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:17:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:19:33.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:17:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:19:33.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:17:27.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-31 17:19:33.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search/mobile_search_clients_engines_sources_daily/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search/mobile_search_clients_engines_sources_daily/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search/mobile_search_clients_engines_sources_daily/schema.yaml 2024-05-31 17:17:27.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search/mobile_search_clients_engines_sources_daily/schema.yaml 1970-01-01 00:00:00.000000000 +0000 @@ -1,120 +0,0 @@ -fields: -- name: submission_date - type: DATE - mode: NULLABLE -- name: client_id - type: STRING - mode: NULLABLE -- name: engine - type: STRING - mode: NULLABLE -- name: source - type: STRING - mode: NULLABLE -- name: app_name - type: STRING - mode: NULLABLE -- name: search_count - type: INTEGER - mode: NULLABLE -- name: organic - type: INTEGER - mode: NULLABLE -- name: tagged_sap - type: INTEGER - mode: NULLABLE -- name: tagged_follow_on - type: INTEGER - mode: NULLABLE -- name: ad_click - type: INTEGER - mode: NULLABLE -- name: search_with_ads - type: INTEGER - mode: NULLABLE -- name: unknown - type: INTEGER - mode: NULLABLE -- name: country - type: STRING - mode: NULLABLE -- name: locale - type: STRING - mode: NULLABLE -- name: app_version - type: STRING - mode: NULLABLE -- name: channel - type: STRING - mode: NULLABLE -- name: os - type: STRING - mode: NULLABLE -- name: os_version - type: STRING - mode: NULLABLE -- name: default_search_engine - type: STRING - mode: NULLABLE -- name: default_search_engine_submission_url - type: STRING - mode: NULLABLE -- name: distribution_id - type: STRING - mode: NULLABLE -- name: profile_creation_date - type: INTEGER - mode: NULLABLE -- name: profile_age_in_days - type: INTEGER - mode: NULLABLE -- name: sample_id - type: INTEGER - mode: NULLABLE -- name: experiments - type: RECORD - mode: REPEATED - fields: - - name: key - type: STRING - mode: NULLABLE - - name: value - type: STRING - mode: NULLABLE -- name: total_uri_count - type: INTEGER - mode: NULLABLE -- name: ad_click_organic - type: INTEGER - mode: NULLABLE -- name: normalized_engine - type: STRING - mode: NULLABLE -- name: normalized_app_name - type: STRING - mode: NULLABLE -- name: browser_version_info - type: RECORD - mode: NULLABLE - fields: - - name: version - type: STRING - mode: NULLABLE - - name: major_version - type: NUMERIC - mode: NULLABLE - - name: minor_version - type: NUMERIC - mode: NULLABLE - - name: patch_revision - type: NUMERIC - mode: NULLABLE - - name: is_major_release - type: BOOLEAN - mode: NULLABLE -- name: sap - type: INTEGER - mode: NULLABLE -- name: normalized_app_name_os - type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/metadata.yaml 2024-05-31 17:18:25.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/metadata.yaml 2024-05-31 17:29:04.000000000 +0000 @@ -1,30 +1,9 @@ -friendly_name: Mobile Search Clients Daily -description: | - A daily aggregate of baseline and metrics pings that have - searches from each (client_id, engine, source) combination, - partitioned by day. - - Exposed to users as `search.mobile_search_clients_engines_sources_daily`. -owners: -- akomar@mozilla.com -- cmorales@mozilla.com -labels: - schedule: daily - dag: bqetl_mobile_search - owner1: akomar - owner2: cmorales -scheduling: - dag_name: bqetl_mobile_search -bigquery: - time_partitioning: - type: day - field: submission_date - require_partition_filter: true - expiration_days: null - range_partitioning: null - clustering: - fields: - - sample_id +friendly_name: Mobile Search Clients Daily V1 +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null workgroup_access: - role: roles/bigquery.dataViewer members: diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/query.sql 2024-05-31 17:16:49.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/query.sql 2024-05-31 17:19:29.000000000 +0000 @@ -1,6 +1,4 @@ --- Query generated by ./bqetl generate search --- This file doesn't get overwritten by the generator. The generator output needs --- to be written to this file manually. +-- Query generated by bigquery-etl/search/mobile_search_clients_daily.py -- -- Older versions separate source and engine with an underscore instead of period -- Return array of form [source, engine] if key is valid, empty array otherwise @@ -498,7 +496,7 @@ metrics.counter.browser_total_uri_count, client_info.locale, FROM - org_mozilla_ios_klar.metrics AS org_mozilla_klar_metrics + org_mozilla_ios_klar.metrics AS org_mozilla_ios_klar_metrics ), fenix_baseline AS ( SELECT @@ -722,6 +720,12 @@ SUBSTR(search.key, STRPOS(search.key, '.') + 1), search.search_type ) + WHEN search.search_type = 'search-with-ads' + THEN IF( + REGEXP_CONTAINS(search.key, '\\.'), + SUBSTR(search.key, STRPOS(search.key, '.') + 1), + search.search_type + ) ELSE search.search_type END AS source, search.value AS search_count, @@ -774,6 +778,8 @@ CASE WHEN search_type = 'ad-click' THEN IF(STARTS_WITH(source, 'in-content.organic'), 'ad-click-organic', search_type) + WHEN search_type = 'search-with-ads' + THEN IF(STARTS_WITH(source, 'in-content.organic'), 'search-with-ads-organic', search_type) WHEN STARTS_WITH(source, 'in-content.sap.') THEN 'tagged-sap' WHEN REGEXP_CONTAINS(source, '^in-content.*-follow-on') @@ -856,6 +862,15 @@ ) ) AS search_with_ads, SUM( + IF( + search_type != 'search-with-ads-organic' + OR engine IS NULL + OR search_count > 10000, + 0, + search_count + ) + ) AS search_with_ads_organic, + SUM( IF(search_type != 'unknown' OR engine IS NULL OR search_count > 10000, 0, search_count) ) AS unknown, udf.mode_last(ARRAY_AGG(country)) AS country, @@ -874,7 +889,6 @@ ANY_VALUE(sample_id) AS sample_id, udf.map_mode_last(ARRAY_CONCAT_AGG(experiments)) AS experiments, SUM(total_uri_count) AS total_uri_count, - CAST(NULL AS STRING) AS normalized_engine FROM combined_search_clients WHERE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-05-31 17:16:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-05-31 17:25:06.000000000 +0000 @@ -1,9 +1,7 @@ fields: - name: postal_district_code type: STRING - mode: REQUIRED - description: One-character Canadian postal district code. + mode: NULLABLE - name: province_code type: STRING mode: NULLABLE - description: Two-character Canadian province/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-05-31 17:16:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-05-31 17:25:06.000000000 +0000 @@ -1,47 +1,28 @@ fields: - name: name - description: Official country name per ISO 3166 type: STRING - mode: REQUIRED + mode: NULLABLE - name: code - description: ISO 3166 alpha-2 country code type: STRING - mode: REQUIRED + mode: NULLABLE - name: code_3 - description: ISO 3166 alpha-3 country code type: STRING - mode: REQUIRED + mode: NULLABLE - name: region_name - description: Region name. These are based on the UN Statistics Division standard - country or area codes for statistical use (M49), but with the "Americas" region - split into "North America" and "South America". type: STRING - mode: REQUIRED + mode: NULLABLE - name: subregion_name - description: Sub-region name. These are based on UN Statistics Division standard - country or area codes for statistical use (M49), but with the "Latin America and the - Caribbean" and "Sub-Saharan Africa" sub-regions split into more specific - sub-regions. type: STRING - mode: REQUIRED + mode: NULLABLE - name: pocket_available_on_newtab - description: Whether Pocket is available on the newtab page in this country. Note - that Pocket might only be available in certain locales/languages within a country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: mozilla_vpn_available - description: Whether Mozilla VPN is available in this country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: sponsored_tiles_available_on_newtab - description: Whether sponsored tiles are available on the newtab page in this country. - Note that Pocket might only be available in certain locales/languages within a - country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: ads_value_tier - description: Lowercase label detailing the monetary value tier that Mozilla Ads - assign to that region based on market size and our existing products, e.g., tier - 1, tier 2, etc. type: STRING - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-05-31 17:16:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-05-31 17:25:06.000000000 +0000 @@ -1,10 +1,7 @@ fields: - name: name - description: An alias for a country's name (including misspellings and alternate - encodings). type: STRING - mode: REQUIRED + mode: NULLABLE - name: code - description: ISO 3166 alpha-2 country code type: STRING - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-05-31 17:16:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-05-31 17:25:06.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: incident + mode: NULLABLE +- name: incident type: STRING -- mode: NULLABLE - name: description + mode: NULLABLE +- name: description type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING -- mode: NULLABLE - name: product + mode: NULLABLE +- name: product type: STRING -- mode: NULLABLE - name: version + mode: NULLABLE +- name: version type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-05-31 17:16:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-05-31 17:25:06.000000000 +0000 @@ -1,27 +1,16 @@ fields: -- mode: NULLABLE - description: Hex value assigned to the TLS cipher, in format like "0x00,0x84"; note - some values are ranges or contain wildcards - name: value +- name: value type: STRING -- mode: NULLABLE - description: Human-readable name of the TLS cipher - name: description + mode: NULLABLE +- name: description type: STRING -- mo ```⚠️ Only part of the diff is displayed.
mobile_search_clients_dailyis based on a SQL generator template which needed to be updated and should fix the failing SQL testsChecklist for reviewer:
<username>:<branch>of the fork as parameter. The parameter will also show up in the logs of themanual-trigger-required-for-forkCI task together with more detailed instructions.For modifications to schemas in restricted namespaces (see
CODEOWNERS):┆Issue is synchronized with this Jira Task