search

mozilla / bigquery-etl

Bigquery ETL
https://mozilla.github.io/bigquery-etl
Mozilla Public License 2.0
241 stars 98 forks source link

Newtab table default UI pocket #5784

Closed m-d-bowerman closed 2 weeks ago

m-d-bowerman commented 2 weeks ago

Checklist for reviewer:

For modifications to schemas in restricted namespaces (see CODEOWNERS):

┆Issue is synchronized with this Jira Task

dataops-ci-bot commented 2 weeks ago

Integration report for "Merge branch 'main' into newtab_table_defaultUI_pocketID"

sql.diff

Click to expand! ```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql 2024-06-13 14:45:38.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql 2024-06-13 14:45:22.000000000 +0000 @@ -20,16 +20,6 @@ WHERE DATE(submission_timestamp) = @submission_date AND category IN ('newtab', 'topsites', 'newtab.search', 'newtab.search.ad', 'pocket') - AND name IN ( - 'closed', - 'opened', - 'impression', - 'issued', - 'click', - 'save', - 'topic_click', - 'dismiss' - ) ), visit_metadata AS ( SELECT @@ -54,6 +44,7 @@ ANY_VALUE(metrics.string.newtab_homepage_category) AS newtab_homepage_category, ANY_VALUE(metrics.string.newtab_newtab_category) AS newtab_newtab_category, ANY_VALUE(metrics.boolean.newtab_search_enabled) AS newtab_search_enabled, + ANY_VALUE(metrics.boolean.newtab_weather_enabled) AS weather_widget_enabled, ANY_VALUE(metrics.quantity.topsites_rows) AS topsites_rows, ANY_VALUE(metrics.string_list.newtab_blocked_sponsors) AS newtab_blocked_sponsors, ANY_VALUE(ping_info.experiments) AS experiments, @@ -110,6 +101,7 @@ events_unnested WHERE event_category IN ('newtab.search', 'newtab.search.ad') + AND event_name IN ('click', 'impression', 'issued') GROUP BY newtab_visit_id, search_engine, @@ -180,6 +172,7 @@ ) WHERE event_category = 'topsites' + AND event_name IN ('dismiss', 'click', 'impression') GROUP BY newtab_visit_id, topsite_tile_position, @@ -218,6 +211,8 @@ SELECT mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS pocket_story_position, + mozfun.map.get_key(event_details, "tile_id") AS pocket_tile_id, + mozfun.map.get_key(event_details, "recommendation_id") AS pocket_recommendation_id, COUNTIF(event_name = 'save') AS pocket_saves, COUNTIF(event_name = 'click') AS pocket_clicks, COUNTIF(event_name = 'impression') AS pocket_impressions, @@ -249,9 +244,12 @@ events_unnested WHERE event_category = 'pocket' + AND event_name IN ('impression', 'click', 'save') GROUP BY newtab_visit_id, - pocket_story_position + pocket_story_position, + pocket_tile_id, + pocket_recommendation_id ), pocket_summary AS ( SELECT @@ -259,6 +257,8 @@ ARRAY_AGG( STRUCT( pocket_story_position, + pocket_tile_id, + pocket_recommendation_id, pocket_impressions, sponsored_pocket_impressions, organic_pocket_impressions, @@ -275,6 +275,99 @@ GROUP BY newtab_visit_id ), +wallpaper_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + mozfun.map.get_key(event_details, "selected_wallpaper") AS wallpaper_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_click') AS wallpaper_clicks, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "true" + ) AS wallpaper_clicks_had_previous_wallpaper, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "false" + ) AS wallpaper_clicks_first_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_category_click') AS wallpaper_category_clicks, + COUNTIF(event_name = 'wallpaper_highlight_dismissed') AS wallpaper_highlight_dismissals, + COUNTIF(event_name = 'wallpaper_highlight_cta_click') AS wallpaper_highlight_cta_clicks + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'wallpaper_click', + 'wallpaper_category_click', + 'wallpaper_highlight_cta_clicks', + 'wallpaper_highlight_dismissed' + ) + GROUP BY + newtab_visit_id, + wallpaper_selected_wallpaper +), +wallpaper_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + wallpaper_selected_wallpaper, + wallpaper_clicks, + wallpaper_clicks_had_previous_wallpaper, + wallpaper_clicks_first_selected_wallpaper, + wallpaper_category_clicks, + wallpaper_highlight_dismissals, + wallpaper_highlight_cta_clicks + ) + ) AS wallpaper_interactions + FROM + wallpaper_events + GROUP BY + newtab_visit_id +), +weather_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + COUNTIF(event_name = 'weather_impression') AS weather_widget_impressions, + COUNTIF(event_name = 'weather_open_provider_url') AS weather_widget_clicks, + COUNTIF(event_name = 'weather_load_error') AS weather_widget_load_errors, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "detailed" + ) AS weather_widget_change_display_to_detailed, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "simple" + ) AS weather_widget_change_display_to_simple + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'weather_impression', + 'weather_open_provider_url', + 'weather_load_error', + 'weather_change_display' + ) + GROUP BY + newtab_visit_id +), +weather_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + weather_widget_impressions, + weather_widget_clicks, + weather_widget_load_errors, + weather_widget_change_display_to_detailed, + weather_widget_change_display_to_simple + ) + ) AS weather_interactions + FROM + weather_events + GROUP BY + newtab_visit_id +), combined_newtab_activity AS ( SELECT * @@ -289,6 +382,12 @@ LEFT JOIN pocket_summary USING (newtab_visit_id) + LEFT JOIN + wallpaper_summary + USING (newtab_visit_id) + LEFT JOIN + weather_summary + USING (newtab_visit_id) WHERE -- Keep only rows with interactions, unless we receive a valid newtab.opened event. -- This is meant to drop only interactions that only have a newtab.closed event on the same partition @@ -297,6 +396,8 @@ OR search_interactions IS NOT NULL OR topsite_tile_interactions IS NOT NULL OR pocket_interactions IS NOT NULL + OR wallpaper_interactions IS NOT NULL + OR weather_interactions IS NOT NULL ), client_profile_info AS ( SELECT @@ -311,7 +412,15 @@ client_id ) SELECT - * + *, + CASE + WHEN ( + (newtab_open_source = "about:home" AND newtab_homepage_category = "enabled") + OR (newtab_open_source = "about:newtab" AND newtab_newtab_category = "enabled") + ) + THEN "default" + ELSE "non-default" + END AS newtab_default_ui, FROM combined_newtab_activity LEFT JOIN diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml 2024-06-13 14:45:38.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml 2024-06-13 14:45:22.000000000 +0000 @@ -215,6 +215,12 @@ - mode: NULLABLE name: organic_pocket_saves type: INTEGER + - name: pocket_tile_id + type: STRING + mode: NULLABLE + - name: pocket_recommendation_id + type: STRING + mode: NULLABLE mode: REPEATED name: pocket_interactions type: RECORD @@ -224,3 +230,53 @@ - mode: NULLABLE name: activity_segment type: STRING +- mode: NULLABLE + name: newtab_default_ui + type: STRING +- name: weather_widget_enabled + type: BOOLEAN + mode: NULLABLE +- name: wallpaper_interactions + type: RECORD + mode: REPEATED + fields: + - name: wallpaper_selected_wallpaper + type: STRING + mode: NULLABLE + - name: wallpaper_clicks + type: INTEGER + mode: NULLABLE + - name: wallpaper_clicks_had_previous_wallpaper + type: INTEGER + mode: NULLABLE + - name: wallpaper_clicks_first_selected_wallpaper + type: INTEGER + mode: NULLABLE + - name: wallpaper_category_clicks + type: INTEGER + mode: NULLABLE + - name: wallpaper_highlight_dismissals + type: INTEGER + mode: NULLABLE + - name: wallpaper_highlight_cta_clicks + type: INTEGER + mode: NULLABLE +- name: weather_interactions + type: RECORD + mode: REPEATED + fields: + - name: weather_widget_impressions + type: INTEGER + mode: NULLABLE + - name: weather_widget_clicks + type: INTEGER + mode: NULLABLE + - name: weather_widget_load_errors + type: INTEGER + mode: NULLABLE + - name: weather_widget_change_display_to_detailed + type: INTEGER + mode: NULLABLE + - name: weather_widget_change_display_to_simple + type: INTEGER + mode: NULLABLE ```

Link to full diff

dataops-ci-bot commented 2 weeks ago

Integration report for "Add new weather, wallpaper telemetry"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/main-generated-sql/dags/: bqetl_census_feed.py Only in /tmp/workspace/main-generated-sql/dags/: bqetl_cloudflare.py Only in /tmp/workspace/main-generated-sql/dags/: bqetl_desktop_conv_evnt_categorization.py Only in /tmp/workspace/generated-sql/dags/: bqetl_mobile_engagement_model.py Only in /tmp/workspace/main-generated-sql/dags/: bqetl_mobile_kpi_metrics.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_acoustic_contact_export.py /tmp/workspace/generated-sql/dags/bqetl_acoustic_contact_export.py --- /tmp/workspace/main-generated-sql/dags/bqetl_acoustic_contact_export.py 2024-06-13 14:45:29.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_acoustic_contact_export.py 2024-06-13 14:58:27.000000000 +0000 @@ -99,6 +99,20 @@ depends_on_past=False, ) + with TaskGroup( + "acoustic_external__contact_raw__v1_external", + ) as acoustic_external__contact_raw__v1_external: + ExternalTaskMarker( + task_id="bqetl_braze__wait_for_acoustic_external__contact_raw__v1", + external_dag_id="bqetl_braze", + external_task_id="wait_for_acoustic_external__contact_raw__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=82800)).isoformat() }}", + ) + + acoustic_external__contact_raw__v1_external.set_upstream( + acoustic_external__contact_raw__v1 + ) + acoustic_derived__contact__v1.set_upstream(acoustic_external__contact_raw__v1) acoustic_derived__contact_current_snapshot__v1.set_upstream( diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_acoustic_raw_recipient_export.py /tmp/workspace/generated-sql/dags/bqetl_acoustic_raw_recipient_export.py --- /tmp/workspace/main-generated-sql/dags/bqetl_acoustic_raw_recipient_export.py 2024-06-13 14:45:29.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_acoustic_raw_recipient_export.py 2024-06-13 14:58:26.000000000 +0000 @@ -87,6 +87,20 @@ depends_on_past=False, ) + with TaskGroup( + "acoustic_external__raw_recipient_raw__v1_external", + ) as acoustic_external__raw_recipient_raw__v1_external: + ExternalTaskMarker( + task_id="bqetl_braze__wait_for_acoustic_external__raw_recipient_raw__v1", + external_dag_id="bqetl_braze", + external_task_id="wait_for_acoustic_external__raw_recipient_raw__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=82800)).isoformat() }}", + ) + + acoustic_external__raw_recipient_raw__v1_external.set_upstream( + acoustic_external__raw_recipient_raw__v1 + ) + acoustic_derived__raw_recipient__v1.set_upstream( acoustic_external__raw_recipient_raw__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_analytics_aggregations.py /tmp/workspace/generated-sql/dags/bqetl_analytics_aggregations.py --- /tmp/workspace/main-generated-sql/dags/bqetl_analytics_aggregations.py 2024-06-13 14:45:29.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_analytics_aggregations.py 2024-06-13 14:58:28.000000000 +0000 @@ -54,6 +54,20 @@ tags=tags, ) as dag: + wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", + external_dag_id="bqetl_analytics_tables", + external_task_id="checks__fail_fenix_derived__firefox_android_clients__v1", + execution_delta=datetime.timedelta(seconds=8100), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + wait_for_checks__fail_telemetry_derived__unified_metrics__v1 = ExternalTaskSensor( task_id="wait_for_checks__fail_telemetry_derived__unified_metrics__v1", external_dag_id="bqetl_unified", @@ -126,36 +140,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 = ExternalTaskSensor( - task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", + wait_for_telemetry_derived__clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_telemetry_derived__clients_last_seen__v1", external_dag_id="bqetl_main_summary", - external_task_id="checks__fail_telemetry_derived__clients_last_seen__v2", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 = ( - ExternalTaskSensor( - task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - external_dag_id="bqetl_analytics_tables", - external_task_id="checks__fail_fenix_derived__firefox_android_clients__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - ) - - wait_for_fenix_derived__metrics_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_fenix_derived__metrics_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="fenix.fenix_derived__metrics_clients_last_seen__v1", + external_task_id="telemetry_derived__clients_last_seen__v1", execution_delta=datetime.timedelta(seconds=8100), check_existence=True, mode="reschedule", @@ -190,46 +178,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_firefox_ios_derived__metrics_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_firefox_ios_derived__metrics_clients_last_seen__v1", + wait_for_fenix_derived__clients_last_seen_joined__v1 = ExternalTaskSensor( + task_id="wait_for_fenix_derived__clients_last_seen_joined__v1", external_dag_id="bqetl_glean_usage", - external_task_id="firefox_ios.firefox_ios_derived__metrics_clients_last_seen__v1", + external_task_id="fenix.fenix_derived__clients_last_seen_joined__v1", execution_delta=datetime.timedelta(seconds=8100), check_existence=True, mode="reschedule", @@ -238,10 +190,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1", + wait_for_search_derived__mobile_search_clients_daily__v1 = ExternalTaskSensor( + task_id="wait_for_search_derived__mobile_search_clients_daily__v1", + external_dag_id="bqetl_mobile_search", + external_task_id="search_derived__mobile_search_clients_daily__v1", execution_delta=datetime.timedelta(seconds=8100), check_existence=True, mode="reschedule", @@ -250,10 +202,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", + wait_for_firefox_ios_derived__clients_last_seen_joined__v1 = ExternalTaskSensor( + task_id="wait_for_firefox_ios_derived__clients_last_seen_joined__v1", external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1", + external_task_id="firefox_ios.firefox_ios_derived__clients_last_seen_joined__v1", execution_delta=datetime.timedelta(seconds=8100), check_existence=True, mode="reschedule", @@ -262,22 +214,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", + wait_for_focus_android_derived__clients_last_seen_joined__v1 = ExternalTaskSensor( + task_id="wait_for_focus_android_derived__clients_last_seen_joined__v1", external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_focus_android_derived__metrics_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_focus_android_derived__metrics_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_android.focus_android_derived__metrics_clients_last_seen__v1", + external_task_id="focus_android.focus_android_derived__clients_last_seen_joined__v1", execution_delta=datetime.timedelta(seconds=8100), check_existence=True, mode="reschedule", @@ -298,58 +238,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_ios.checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_focus_ios_derived__metrics_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_focus_ios_derived__metrics_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="focus_ios.focus_ios_derived__metrics_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_android.checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_klar_android_derived__metrics_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_klar_android_derived__metrics_clients_last_seen__v1", - external_dag_id="bqetl_glean_usage", - external_task_id="klar_android.klar_android_derived__metrics_clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - - wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", + wait_for_focus_ios_derived__clients_last_seen_joined__v1 = ExternalTaskSensor( + task_id="wait_for_focus_ios_derived__clients_last_seen_joined__v1", external_dag_id="bqetl_glean_usage", - external_task_id="klar_ios.checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1", + external_task_id="focus_ios.focus_ios_derived__clients_last_seen_joined__v1", execution_delta=datetime.timedelta(seconds=8100), check_existence=True, mode="reschedule", @@ -358,10 +250,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_klar_ios_derived__metrics_clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_klar_ios_derived__metrics_clients_last_seen__v1", + wait_for_klar_ios_derived__clients_last_seen_joined__v1 = ExternalTaskSensor( + task_id="wait_for_klar_ios_derived__clients_last_seen_joined__v1", external_dag_id="bqetl_glean_usage", - external_task_id="klar_ios.klar_ios_derived__metrics_clients_last_seen__v1", + external_task_id="klar_ios.klar_ios_derived__clients_last_seen_joined__v1", execution_delta=datetime.timedelta(seconds=8100), check_existence=True, mode="reschedule", @@ -396,16 +288,19 @@ ) ) - wait_for_telemetry_derived__clients_last_seen__v1 = ExternalTaskSensor( - task_id="wait_for_telemetry_derived__clients_last_seen__v1", - external_dag_id="bqetl_main_summary", - external_task_id="telemetry_derived__clients_last_seen__v1", - execution_delta=datetime.timedelta(seconds=8100), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", + active_users_aggregates_attribution_v1 = bigquery_etl_query( + task_id="active_users_aggregates_attribution_v1", + destination_table="active_users_aggregates_attribution_v1", + dataset_id="telemetry_derived", + project_id="moz-fx-data-shared-prod", + owner="lvargas@mozilla.com", + email=[ + "gkaberere@mozilla.com", + "lvargas@mozilla.com", + "telemetry-alerts@mozilla.com", + ], + date_partition_parameter="submission_date", + depends_on_past=False, ) active_users_aggregates_device_v1 = bigquery_etl_query( @@ -423,6 +318,20 @@ depends_on_past=False, ) + with TaskGroup( + "active_users_aggregates_device_v1_external", + ) as active_users_aggregates_device_v1_external: + ExternalTaskMarker( + task_id="bqetl_search_dashboard__wait_for_active_users_aggregates_device_v1", + external_dag_id="bqetl_search_dashboard", + external_task_id="wait_for_active_users_aggregates_device_v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", + ) + + active_users_aggregates_device_v1_external.set_upstream( + active_users_aggregates_device_v1 + ) + active_users_aggregates_v1 = bigquery_etl_query( task_id="active_users_aggregates_v1", destination_table="active_users_aggregates_v1", @@ -440,7 +349,7 @@ checks__fail_fenix_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__fail_fenix_derived__active_users_aggregates__v3", - source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + source_table="active_users_aggregates_v3", dataset_id="fenix_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=True, @@ -451,27 +360,13 @@ "telemetry-alerts@mozilla.com", ], depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], + parameters=["submission_date:DATE:{{ds}}"], retries=0, ) - with TaskGroup( - "checks__fail_fenix_derived__active_users_aggregates__v3_external", - ) as checks__fail_fenix_derived__active_users_aggregates__v3_external: - ExternalTaskMarker( - task_id="bqetl_search_dashboard__wait_for_checks__fail_fenix_derived__active_users_aggregates__v3", - external_dag_id="bqetl_search_dashboard", - external_task_id="wait_for_checks__fail_fenix_derived__active_users_aggregates__v3", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", - ) - - checks__fail_fenix_derived__active_users_aggregates__v3_external.set_upstream( - checks__fail_fenix_derived__active_users_aggregates__v3 - ) - checks__fail_firefox_ios_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__fail_firefox_ios_derived__active_users_aggregates__v3", - source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + source_table="active_users_aggregates_v3", dataset_id="firefox_ios_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=True, @@ -482,27 +377,13 @@ "telemetry-alerts@mozilla.com", ], depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], + parameters=["submission_date:DATE:{{ds}}"], retries=0, ) - with TaskGroup( - "checks__fail_firefox_ios_derived__active_users_aggregates__v3_external", - ) as checks__fail_firefox_ios_derived__active_users_aggregates__v3_external: - ExternalTaskMarker( - task_id="bqetl_search_dashboard__wait_for_checks__fail_firefox_ios_derived__active_users_aggregates__v3", - external_dag_id="bqetl_search_dashboard", - external_task_id="wait_for_checks__fail_firefox_ios_derived__active_users_aggregates__v3", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", - ) - - checks__fail_firefox_ios_derived__active_users_aggregates__v3_external.set_upstream( - checks__fail_firefox_ios_derived__active_users_aggregates__v3 - ) - checks__fail_focus_android_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__fail_focus_android_derived__active_users_aggregates__v3", - source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + source_table="active_users_aggregates_v3", dataset_id="focus_android_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=True, @@ -513,27 +394,13 @@ "telemetry-alerts@mozilla.com", ], depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], + parameters=["submission_date:DATE:{{ds}}"], retries=0, ) - with TaskGroup( - "checks__fail_focus_android_derived__active_users_aggregates__v3_external", - ) as checks__fail_focus_android_derived__active_users_aggregates__v3_external: - ExternalTaskMarker( - task_id="bqetl_search_dashboard__wait_for_checks__fail_focus_android_derived__active_users_aggregates__v3", - external_dag_id="bqetl_search_dashboard", - external_task_id="wait_for_checks__fail_focus_android_derived__active_users_aggregates__v3", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", - ) - - checks__fail_focus_android_derived__active_users_aggregates__v3_external.set_upstream( - checks__fail_focus_android_derived__active_users_aggregates__v3 - ) - checks__fail_focus_ios_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__fail_focus_ios_derived__active_users_aggregates__v3", - source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + source_table="active_users_aggregates_v3", dataset_id="focus_ios_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=True, @@ -544,58 +411,13 @@ "telemetry-alerts@mozilla.com", ], depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], + parameters=["submission_date:DATE:{{ds}}"], retries=0, ) - with TaskGroup( - "checks__fail_focus_ios_derived__active_users_aggregates__v3_external", - ) as checks__fail_focus_ios_derived__active_users_aggregates__v3_external: - ExternalTaskMarker( - task_id="bqetl_search_dashboard__wait_for_checks__fail_focus_ios_derived__active_users_aggregates__v3", - external_dag_id="bqetl_search_dashboard", - external_task_id="wait_for_checks__fail_focus_ios_derived__active_users_aggregates__v3", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", - ) - - checks__fail_focus_ios_derived__active_users_aggregates__v3_external.set_upstream( - checks__fail_focus_ios_derived__active_users_aggregates__v3 - ) - - checks__fail_klar_android_derived__active_users_aggregates__v3 = bigquery_dq_check( - task_id="checks__fail_klar_android_derived__active_users_aggregates__v3", - source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="klar_android_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=True, - owner="lvargas@mozilla.com", - email=[ - "gkaberere@mozilla.com", - "lvargas@mozilla.com", - "telemetry-alerts@mozilla.com", - ], - depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], - retries=0, - ) - - with TaskGroup( - "checks__fail_klar_android_derived__active_users_aggregates__v3_external", - ) as checks__fail_klar_android_derived__active_users_aggregates__v3_external: - ExternalTaskMarker( - task_id="bqetl_search_dashboard__wait_for_checks__fail_klar_android_derived__active_users_aggregates__v3", - external_dag_id="bqetl_search_dashboard", - external_task_id="wait_for_checks__fail_klar_android_derived__active_users_aggregates__v3", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", - ) - - checks__fail_klar_android_derived__active_users_aggregates__v3_external.set_upstream( - checks__fail_klar_android_derived__active_users_aggregates__v3 - ) - checks__fail_klar_ios_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__fail_klar_ios_derived__active_users_aggregates__v3", - source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + source_table="active_users_aggregates_v3", dataset_id="klar_ios_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=True, @@ -606,27 +428,13 @@ "telemetry-alerts@mozilla.com", ], depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], + parameters=["submission_date:DATE:{{ds}}"], retries=0, ) - with TaskGroup( - "checks__fail_klar_ios_derived__active_users_aggregates__v3_external", - ) as checks__fail_klar_ios_derived__active_users_aggregates__v3_external: - ExternalTaskMarker( - task_id="bqetl_search_dashboard__wait_for_checks__fail_klar_ios_derived__active_users_aggregates__v3", - external_dag_id="bqetl_search_dashboard", - external_task_id="wait_for_checks__fail_klar_ios_derived__active_users_aggregates__v3", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", - ) - - checks__fail_klar_ios_derived__active_users_aggregates__v3_external.set_upstream( - checks__fail_klar_ios_derived__active_users_aggregates__v3 - ) - checks__warn_fenix_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__warn_fenix_derived__active_users_aggregates__v3", - source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + source_table="active_users_aggregates_v3", dataset_id="fenix_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=False, @@ -637,13 +445,14 @@ "telemetry-alerts@mozilla.com", ], depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], + parameters=["submission_date:DATE:{{ds}}"], retries=0, ) - checks__warn_firefox_desktop_derived__active_users_aggregates__v3 = bigquery_dq_check( + checks__warn_firefox_desktop_derived__active_users_aggregates__v3 = ( + bigquery_dq_check( task_id="checks__warn_firefox_desktop_derived__active_users_aggregates__v3", - source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + source_table="active_users_aggregates_v3", dataset_id="firefox_desktop_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=False, @@ -654,13 +463,14 @@ "telemetry-alerts@mozilla.com", ], depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], + parameters=["submission_date:DATE:{{ds}}"], retries=0, ) + ) checks__warn_firefox_ios_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__warn_firefox_ios_derived__active_users_aggregates__v3", - source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + source_table="active_users_aggregates_v3", dataset_id="firefox_ios_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=False, @@ -671,13 +481,13 @@ "telemetry-alerts@mozilla.com", ], depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], + parameters=["submission_date:DATE:{{ds}}"], retries=0, ) checks__warn_focus_android_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__warn_focus_android_derived__active_users_aggregates__v3", - source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + source_table="active_users_aggregates_v3", dataset_id="focus_android_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=False, @@ -688,13 +498,13 @@ "telemetry-alerts@mozilla.com", ], depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], + parameters=["submission_date:DATE:{{ds}}"], retries=0, ) checks__warn_focus_ios_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__warn_focus_ios_derived__active_users_aggregates__v3", - source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + source_table="active_users_aggregates_v3", dataset_id="focus_ios_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=False, @@ -705,30 +515,13 @@ "telemetry-alerts@mozilla.com", ], depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], - retries=0, - ) - - checks__warn_klar_android_derived__active_users_aggregates__v3 = bigquery_dq_check( - task_id="checks__warn_klar_android_derived__active_users_aggregates__v3", - source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="klar_android_derived", - project_id="moz-fx-data-shared-prod", - is_dq_check_fail=False, - owner="lvargas@mozilla.com", - email=[ - "gkaberere@mozilla.com", - "lvargas@mozilla.com", - "telemetry-alerts@mozilla.com", - ], - depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], + parameters=["submission_date:DATE:{{ds}}"], retries=0, ) checks__warn_klar_ios_derived__active_users_aggregates__v3 = bigquery_dq_check( task_id="checks__warn_klar_ios_derived__active_users_aggregates__v3", - source_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + source_table="active_users_aggregates_v3", dataset_id="klar_ios_derived", project_id="moz-fx-data-shared-prod", is_dq_check_fail=False, @@ -739,13 +532,13 @@ "telemetry-alerts@mozilla.com", ], depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], + parameters=["submission_date:DATE:{{ds}}"], retries=0, ) fenix_active_users_aggregates = bigquery_etl_query( task_id="fenix_active_users_aggregates", - destination_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + destination_table="active_users_aggregates_v3", dataset_id="fenix_derived", project_id="moz-fx-data-shared-prod", owner="lvargas@mozilla.com", @@ -754,14 +547,13 @@ "lvargas@mozilla.com", "telemetry-alerts@mozilla.com", ], - date_partition_parameter=None, + date_partition_parameter="submission_date", depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], ) firefox_desktop_active_users_aggregates = bigquery_etl_query( task_id="firefox_desktop_active_users_aggregates", - destination_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + destination_table="active_users_aggregates_v3", dataset_id="firefox_desktop_derived", project_id="moz-fx-data-shared-prod", owner="lvargas@mozilla.com", @@ -770,28 +562,13 @@ "lvargas@mozilla.com", "telemetry-alerts@mozilla.com", ], - date_partition_parameter=None, + date_partition_parameter="submission_date", depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], - ) - - with TaskGroup( - "firefox_desktop_active_users_aggregates_external", - ) as firefox_desktop_active_users_aggregates_external: - ExternalTaskMarker( - task_id="bqetl_search_dashboard__wait_for_firefox_desktop_active_users_aggregates", - external_dag_id="bqetl_search_dashboard", - external_task_id="wait_for_firefox_desktop_active_users_aggregates", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=85500)).isoformat() }}", - ) - - firefox_desktop_active_users_aggregates_external.set_upstream( - firefox_desktop_active_users_aggregates ) firefox_ios_active_users_aggregates = bigquery_etl_query( task_id="firefox_ios_active_users_aggregates", - destination_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + destination_table="active_users_aggregates_v3", dataset_id="firefox_ios_derived", project_id="moz-fx-data-shared-prod", owner="lvargas@mozilla.com", @@ -800,14 +577,13 @@ "lvargas@mozilla.com", "telemetry-alerts@mozilla.com", ], - date_partition_parameter=None, + date_partition_parameter="submission_date", depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], ) focus_android_active_users_aggregates = bigquery_etl_query( task_id="focus_android_active_users_aggregates", - destination_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + destination_table="active_users_aggregates_v3", dataset_id="focus_android_derived", project_id="moz-fx-data-shared-prod", owner="lvargas@mozilla.com", @@ -816,14 +592,13 @@ "lvargas@mozilla.com", "telemetry-alerts@mozilla.com", ], - date_partition_parameter=None, + date_partition_parameter="submission_date", depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], ) focus_ios_active_users_aggregates = bigquery_etl_query( task_id="focus_ios_active_users_aggregates", - destination_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + destination_table="active_users_aggregates_v3", dataset_id="focus_ios_derived", project_id="moz-fx-data-shared-prod", owner="lvargas@mozilla.com", @@ -832,30 +607,13 @@ "lvargas@mozilla.com", "telemetry-alerts@mozilla.com", ], - date_partition_parameter=None, - depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], - ) - - klar_android_active_users_aggregates = bigquery_etl_query( - task_id="klar_android_active_users_aggregates", - destination_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="klar_android_derived", - project_id="moz-fx-data-shared-prod", - owner="lvargas@mozilla.com", - email=[ - "gkaberere@mozilla.com", - "lvargas@mozilla.com", - "telemetry-alerts@mozilla.com", - ], - date_partition_parameter=None, + date_partition_parameter="submission_date", depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], ) klar_ios_active_users_aggregates = bigquery_etl_query( task_id="klar_ios_active_users_aggregates", - destination_table='active_users_aggregates_v3${{ macros.ds_format(macros.ds_add(ds, -1), "%Y-%m-%d", "%Y%m%d") }}', + destination_table="active_users_aggregates_v3", dataset_id="klar_ios_derived", project_id="moz-fx-data-shared-prod", owner="lvargas@mozilla.com", @@ -864,9 +622,8 @@ "lvargas@mozilla.com", "telemetry-alerts@mozilla.com", ], - date_partition_parameter=None, + date_partition_parameter="submission_date", depends_on_past=False, - parameters=["submission_date:DATE:{{macros.ds_add(ds, -1)}}"], ) telemetry_derived__cohort_daily_statistics__v1 = bigquery_etl_query( @@ -901,6 +658,14 @@ depends_on_past=False, ) + active_users_aggregates_attribution_v1.set_upstream( + wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 + ) + + active_users_aggregates_attribution_v1.set_upstream( + wait_for_checks__fail_telemetry_derived__unified_metrics__v1 + ) + active_users_aggregates_device_v1.set_upstream( wait_for_checks__fail_telemetry_derived__unified_metrics__v1 ) @@ -945,10 +710,6 @@ focus_ios_active_users_aggregates ) - checks__fail_klar_android_derived__active_users_aggregates__v3.set_upstream( - klar_android_active_users_aggregates - ) - checks__fail_klar_ios_derived__active_users_aggregates__v3.set_upstream( klar_ios_active_users_aggregates ) @@ -978,11 +739,11 @@ ) checks__warn_firefox_desktop_derived__active_users_aggregates__v3.set_upstream( - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 + firefox_desktop_active_users_aggregates ) checks__warn_firefox_desktop_derived__active_users_aggregates__v3.set_upstream( - firefox_desktop_active_users_aggregates + wait_for_telemetry_derived__clients_last_seen__v1 ) checks__warn_firefox_ios_derived__active_users_aggregates__v3.set_upstream( @@ -997,10 +758,6 @@ focus_ios_active_users_aggregates ) - checks__warn_klar_android_derived__active_users_aggregates__v3.set_upstream( - klar_android_active_users_aggregates - ) - checks__warn_klar_ios_derived__active_users_aggregates__v3.set_upstream( klar_ios_active_users_aggregates ) @@ -1010,99 +767,95 @@ ) fenix_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_fenix_derived__baseline_clients_last_seen__v1 - ) - - fenix_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_fenix_nightly_derived__baseline_clients_last_seen__v1 - ) - - fenix_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_fennec_aurora_derived__baseline_clients_last_seen__v1 + wait_for_checks__fail_firefox_ios_derived__clients_activation__v1 ) fenix_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_firefox_beta_derived__baseline_clients_last_seen__v1 + wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 ) fenix_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_firefox_derived__baseline_clients_last_seen__v1 + wait_for_fenix_derived__clients_last_seen_joined__v1 ) fenix_active_users_aggregates.set_upstream( - wait_for_fenix_derived__metrics_clients_last_seen__v1 + wait_for_search_derived__mobile_search_clients_daily__v1 ) firefox_desktop_active_users_aggregates.set_upstream( - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 + wait_for_telemetry_derived__clients_last_seen__v1 ) firefox_ios_active_users_aggregates.set_upstream( - wait_for_checks__fail_firefox_ios_derived__clients_activation__v1 + wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 ) firefox_ios_active_users_aggregates.set_upstream( - wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 + wait_for_checks__fail_firefox_ios_derived__clients_activation__v1 ) firefox_ios_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_ios_fennec_derived__baseline_clients_last_seen__v1 + wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 ) firefox_ios_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + wait_for_firefox_ios_derived__clients_last_seen_joined__v1 ) firefox_ios_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_ios_firefoxbeta_derived__baseline_clients_last_seen__v1 + wait_for_search_derived__mobile_search_clients_daily__v1 ) - firefox_ios_active_users_aggregates.set_upstream( - wait_for_firefox_ios_derived__metrics_clients_last_seen__v1 + focus_android_active_users_aggregates.set_upstream( + wait_for_focus_android_derived__clients_last_seen_joined__v1 ) focus_android_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_focus_beta_derived__baseline_clients_last_seen__v1 + wait_for_search_derived__mobile_search_clients_daily__v1 ) focus_android_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_focus_derived__baseline_clients_last_seen__v1 + wait_for_telemetry_derived__core_clients_last_seen__v1 ) - focus_android_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_focus_nightly_derived__baseline_clients_last_seen__v1 + focus_ios_active_users_aggregates.set_upstream( + wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 ) - focus_android_active_users_aggregates.set_upstream( - wait_for_focus_android_derived__metrics_clients_last_seen__v1 + focus_ios_active_users_aggregates.set_upstream( + wait_for_checks__fail_firefox_ios_derived__clients_activation__v1 ) - focus_android_active_users_aggregates.set_upstream( - wait_for_telemetry_derived__core_clients_last_seen__v1 + focus_ios_active_users_aggregates.set_upstream( + wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 ) focus_ios_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_ios_focus_derived__baseline_clients_last_seen__v1 + wait_for_focus_ios_derived__clients_last_seen_joined__v1 ) focus_ios_active_users_aggregates.set_upstream( - wait_for_focus_ios_derived__metrics_clients_last_seen__v1 + wait_for_search_derived__mobile_search_clients_daily__v1 ) - klar_android_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_klar_derived__baseline_clients_last_seen__v1 + klar_ios_active_users_aggregates.set_upstream( + wait_for_checks__fail_fenix_derived__firefox_android_clients__v1 ) - klar_android_active_users_aggregates.set_upstream( - wait_for_klar_android_derived__metrics_clients_last_seen__v1 + klar_ios_active_users_aggregates.set_upstream( + wait_for_checks__fail_firefox_ios_derived__clients_activation__v1 + ) + + klar_ios_active_users_aggregates.set_upstream( + wait_for_checks__fail_firefox_ios_derived__firefox_ios_clients__v1 ) klar_ios_active_users_aggregates.set_upstream( - wait_for_checks__fail_org_mozilla_ios_klar_derived__baseline_clients_last_seen__v1 + wait_for_klar_ios_derived__clients_last_seen_joined__v1 ) klar_ios_active_users_aggregates.set_upstream( - wait_for_klar_ios_derived__metrics_clients_last_seen__v1 + wait_for_search_derived__mobile_search_clients_daily__v1 ) telemetry_derived__cohort_daily_statistics__v1.set_upstream( diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_analytics_tables.py /tmp/workspace/generated-sql/dags/bqetl_analytics_tables.py --- /tmp/workspace/main-generated-sql/dags/bqetl_analytics_tables.py 2024-06-13 14:45:29.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_analytics_tables.py 2024-06-13 14:58:28.000000000 +0000 @@ -211,10 +211,10 @@ pool="DATA_ENG_EXTERNALTASKSENSOR", ) - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 = ExternalTaskSensor( - task_id="wait_for_checks__fail_telemetry_derived__clients_last_seen__v2", + wait_for_telemetry_derived__clients_last_seen__v1 = ExternalTaskSensor( + task_id="wait_for_telemetry_derived__clients_last_seen__v1", external_dag_id="bqetl_main_summary", - external_task_id="checks__fail_telemetry_derived__clients_last_seen__v2", + external_task_id="telemetry_derived__clients_last_seen__v1", check_existence=True, mode="reschedule", allowed_states=ALLOWED_STATES, @@ -265,13 +265,6 @@ ) ExternalTaskMarker( - task_id="bqetl_mobile_kpi_metrics__wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - external_dag_id="bqetl_mobile_kpi_metrics", - external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mobile_feature_usage__wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", external_dag_id="bqetl_mobile_feature_usage", external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", @@ -291,6 +284,13 @@ external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", ) + ExternalTaskMarker( + task_id="bqetl_mobile_engagement_model__wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", + external_dag_id="bqetl_mobile_engagement_model", + external_task_id="wait_for_checks__fail_fenix_derived__firefox_android_clients__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", + ) + checks__fail_fenix_derived__firefox_android_clients__v1_external.set_upstream( checks__fail_fenix_derived__firefox_android_clients__v1 ) @@ -324,13 +324,6 @@ ) ExternalTaskMarker( - task_id="bqetl_desktop_conv_evnt_categorization__wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", - external_dag_id="bqetl_desktop_conv_evnt_categorization", - external_task_id="wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", - execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=50400)).isoformat() }}", - ) - - ExternalTaskMarker( task_id="bqetl_mozilla_org_derived__wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", external_dag_id="bqetl_mozilla_org_derived", external_task_id="wait_for_checks__fail_telemetry_derived__clients_first_seen__v2", @@ -693,5 +686,5 @@ ) telemetry_derived__clients_first_seen_28_days_later__v1.set_upstream( - wait_for_checks__fail_telemetry_derived__clients_last_seen__v2 + wait_for_telemetry_derived__clients_last_seen__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_braze.py /tmp/workspace/generated-sql/dags/bqetl_braze.py --- /tmp/workspace/main-generated-sql/dags/bqetl_braze.py 2024-06-13 14:45:29.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_braze.py 2024-06-13 14:58:29.000000000 +0000 @@ -50,11 +50,59 @@ with DAG( "bqetl_braze", default_args=default_args, - schedule_interval="30 4,12,20 * * *", + schedule_interval="0 10 * * *", doc_md=docs, tags=tags, ) as dag: + wait_for_subscription_platform_derived__logical_subscriptions_history__v1 = ExternalTaskSensor( + task_id="wait_for_subscription_platform_derived__logical_subscriptions_history__v1", + external_dag_id="bqetl_subplat", + external_task_id="subscription_platform_derived__logical_subscriptions_history__v1", + execution_delta=datetime.timedelta(seconds=29700), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_checks__fail_marketing_suppression_list_derived__main_suppression_list__v1 = ExternalTaskSensor( + task_id="wait_for_checks__fail_marketing_suppression_list_derived__main_suppression_list__v1", + external_dag_id="bqetl_marketing_suppression_list", + external_task_id="checks__fail_marketing_suppression_list_derived__main_suppression_list__v1", + execution_delta=datetime.timedelta(seconds=25200), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_acoustic_external__contact_raw__v1 = ExternalTaskSensor( + task_id="wait_for_acoustic_external__contact_raw__v1", + external_dag_id="bqetl_acoustic_contact_export", + external_task_id="acoustic_external__contact_raw__v1", + execution_delta=datetime.timedelta(seconds=3600), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + + wait_for_acoustic_external__raw_recipient_raw__v1 = ExternalTaskSensor( + task_id="wait_for_acoustic_external__raw_recipient_raw__v1", + external_dag_id="bqetl_acoustic_raw_recipient_export", + external_task_id="acoustic_external__raw_recipient_raw__v1", + execution_delta=datetime.timedelta(seconds=3600), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + braze_derived__newsletters__v1 = bigquery_etl_query( task_id="braze_derived__newsletters__v1", destination_table="newsletters_v1", @@ -103,6 +151,18 @@ sql_file_path="sql/moz-fx-data-shared-prod/braze_derived/subscriptions_map_v1/script.sql", ) + braze_derived__suppressions__v1 = bigquery_etl_query( + task_id="braze_derived__suppressions__v1", + destination_table="suppressions_v1", + dataset_id="braze_derived", + project_id="moz-fx-data-shared-prod", + owner="cbeck@mozilla.com", + email=["cbeck@mozilla.com", "leli@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + task_concurrency=1, + ) + braze_derived__user_profiles__v1 = bigquery_etl_query( task_id="braze_derived__user_profiles__v1", destination_table="user_profiles_v1", @@ -149,7 +209,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__changed_newsletters_sync__v1 = bigquery_etl_query( @@ -162,7 +221,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__changed_products_sync__v1 = bigquery_etl_query( @@ -175,7 +233,18 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], + ) + + braze_external__changed_subscriptions__v1 = bigquery_etl_query( + task_id="braze_external__changed_subscriptions__v1", + destination_table="changed_subscriptions_v1", + dataset_id="braze_external", + project_id="moz-fx-data-shared-prod", + owner="cbeck@mozilla.com", + email=["cbeck@mozilla.com", "leli@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + task_concurrency=1, ) braze_external__changed_users__v1 = bigquery_etl_query( @@ -200,7 +269,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__changed_waitlists_sync__v1 = bigquery_etl_query( @@ -213,7 +281,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__delete_users_sync__v1 = bigquery_etl_query( @@ -226,7 +293,6 @@ date_partition_parameter=None, depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], ) braze_external__users_previous_day_snapshot__v1 = bigquery_etl_query( @@ -344,6 +410,19 @@ retries=0, ) + checks__fail_braze_external__changed_subscriptions__v1 = bigquery_dq_check( + task_id="checks__fail_braze_external__changed_subscriptions__v1", + source_table="changed_subscriptions_v1", + dataset_id="braze_external", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=True, + owner="cbeck@mozilla.com", + email=["cbeck@mozilla.com", "leli@mozilla.com"], + depends_on_past=False, + task_concurrency=1, + retries=0, + ) + checks__warn_braze_external__changed_firefox_subscriptions_sync__v1 = bigquery_dq_check( task_id="checks__warn_braze_external__changed_firefox_subscriptions_sync__v1", source_table="changed_firefox_subscriptions_sync_v1", @@ -354,7 +433,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -368,7 +446,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -382,7 +459,19 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], + retries=0, + ) + + checks__warn_braze_external__changed_subscriptions__v1 = bigquery_dq_check( + task_id="checks__warn_braze_external__changed_subscriptions__v1", + source_table="changed_subscriptions_v1", + dataset_id="braze_external", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=False, + owner="cbeck@mozilla.com", + email=["cbeck@mozilla.com", "leli@mozilla.com"], + depends_on_past=False, + task_concurrency=1, retries=0, ) @@ -409,7 +498,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -423,7 +511,6 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) @@ -437,12 +524,17 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, task_concurrency=1, - arguments=["--append_table", "--noreplace"], retries=0, ) braze_derived__newsletters__v1.set_upstream(checks__fail_braze_derived__users__v1) + braze_derived__products__v1.set_upstream(checks__fail_braze_derived__users__v1) + + braze_derived__products__v1.set_upstream( + wait_for_subscription_platform_derived__logical_subscriptions_history__v1 + ) + braze_derived__subscriptions__v1.set_upstream( checks__fail_braze_derived__subscriptions_map__v1 ) @@ -451,6 +543,10 @@ checks__fail_braze_derived__user_profiles__v1 ) + braze_derived__suppressions__v1.set_upstream( + wait_for_checks__fail_marketing_suppression_list_derived__main_suppression_list__v1 + ) + braze_derived__user_profiles__v1.set_upstream( checks__fail_braze_derived__newsletters__v1 ) @@ -465,10 +561,24 @@ checks__fail_braze_derived__waitlists__v1 ) + braze_derived__users__v1.set_upstream(wait_for_acoustic_external__contact_raw__v1) + + braze_derived__users__v1.set_upstream( + wait_for_acoustic_external__raw_recipient_raw__v1 + ) + + braze_derived__users__v1.set_upstream( + checks__fail_braze_derived__subscriptions_map__v1 + ) + + braze_derived__users__v1.set_upstream( + wait_for_checks__fail_marketing_suppression_list_derived__main_suppression_list__v1 + ) + braze_derived__waitlists__v1.set_upstream(checks__fail_braze_derived__users__v1) braze_external__changed_firefox_subscriptions_sync__v1.set_upstream( - checks__fail_braze_derived__subscriptions__v1 + checks__fail_braze_external__changed_subscriptions__v1 ) braze_external__changed_newsletters_sync__v1.set_upstream( @@ -479,6 +589,10 @@ checks__fail_braze_derived__products__v1 ) + braze_external__changed_subscriptions__v1.set_upstream( + checks__fail_braze_derived__subscriptions__v1 + ) + braze_external__changed_users__v1.set_upstream( braze_external__users_previous_day_snapshot__v2 ) @@ -521,6 +635,14 @@ checks__fail_braze_derived__waitlists__v1.set_upstream(braze_derived__waitlists__v1) + checks__fail_braze_external__changed_subscriptions__v1.set_upstream( + braze_external__changed_subscriptions__v1 + ) + + checks__fail_braze_external__changed_subscriptions__v1.set_upstream( + checks__fail_braze_derived__subscriptions__v1 + ) + checks__warn_braze_external__changed_firefox_subscriptions_sync__v1.set_upstream( braze_external__changed_firefox_subscriptions_sync__v1 ) @@ -533,6 +655,14 @@ braze_external__changed_products_sync__v1 ) + checks__warn_braze_external__changed_subscriptions__v1.set_upstream( + braze_external__changed_subscriptions__v1 + ) + + checks__warn_braze_external__changed_subscriptions__v1.set_upstream( + checks__fail_braze_derived__subscriptions__v1 + ) + checks__warn_braze_external__changed_users__v1.set_upstream( braze_external__changed_users__v1 ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 2024-06-13 14:45:29.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,82 +0,0 @@ -# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py - -from airflow import DAG -from airflow.sensors.external_task import ExternalTaskMarker -from airflow.sensors.external_task import ExternalTaskSensor -from airflow.utils.task_group import TaskGroup -import datetime -from operators.gcp_container_operator import GKEPodOperator -from utils.constants import ALLOWED_STATES, FAILED_STATES -from utils.gcp import bigquery_etl_query, bigquery_dq_check - -docs = """ -### bqetl_census_feed - -Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) - -#### Description - -Loads the desktop conversion event tables -#### Owner - -kwindau@mozilla.com - -#### Tags - -* impact/tier_2 -* repo/bigquery-etl -""" - - -default_args = { - "owner": "kwindau@mozilla.com", - "start_date": datetime.datetime(2024, 6, 10, 0, 0), - "end_date": None, - "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], - "depends_on_past": False, - "retry_delay": datetime.timedelta(seconds=1800), - "email_on_failure": True, - "email_on_retry": False, - "retries": 2, -} - -tags = ["impact/tier_2", "repo/bigquery-etl"] - -with DAG( - "bqetl_census_feed", - default_args=default_args, - schedule_interval="0 17 * * *", - doc_md=docs, - tags=tags, -) as dag: - - wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 = ( - ExternalTaskSensor( - task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", - external_dag_id="bqetl_google_analytics_derived_ga4", - external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v2", - execution_delta=datetime.timedelta(seconds=18000), - check_existence=True, - mode="reschedule", - allowed_states=ALLOWED_STATES, - failed_states=FAILED_STATES, - pool="DATA_ENG_EXTERNALTASKSENSOR", - ) - ) - - mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( - task_id="mozilla_org_derived__ga_desktop_conversions__v1", - destination_table='ga_desktop_conversions_v1${{ macros.ds_format(macros.ds_add(ds, -2), "%Y-%m-%d", "%Y%m%d") }}', - dataset_id="mozilla_org_derived", - project_id="moz-fx-data-shared-prod", - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - parameters=["activity_date:DATE:{{macros.ds_add(ds, -2)}}"] - + ["submission_date:DATE:{{ds}}"], - ) - - mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( - wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 - ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare.py /tmp/workspace/generated-sql/dags/bqetl_cloudflare.py --- /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare.py 2024-06-13 14:45:29.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_cloudflare.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,86 +0,0 @@ -# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py - -from airflow import DAG -from airflow.sensors.external_task import ExternalTaskMarker -from airflow.sensors.external_task import ExternalTaskSensor -from airflow.utils.task_group import TaskGroup -import datetime -from operators.gcp_container_operator import GKEPodOperator -from utils.constants import ALLOWED_STATES, FAILED_STATES -from utils.gcp import bigquery_etl_query, bigquery_dq_check - -docs = """ -### bqetl_cloudflare - -Built from bigquery-etl repo, [`dags/bqetl_cloudflare.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_cloudflare.py) - -#### Description - -Derived tables built on Cloudflare downloaded data - -#### Owner - -kwindau@mozilla.com - -#### Tags - -* impact/tier_3 -* repo/bigquery-etl -""" - - -default_args = { - "owner": "kwindau@mozilla.com", - "start_date": datetime.datetime(2024, 6, 7, 0, 0), - "end_date": None, - "email": ["kwindau@mozilla.com"], - "depends_on_past": False, - "retry_delay": datetime.timedelta(seconds=1800), - "email_on_failure": True, - "email_on_retry": True, - "retries": 2, -} - -tags = ["impact/tier_3", "repo/bigquery-etl"] - -with DAG( - "bqetl_cloudflare", - default_args=default_args, - schedule_interval="0 4 * * *" ```

⚠️ Only part of the diff is displayed.

Link to full diff

dataops-ci-bot commented 2 weeks ago

Integration report for "Update newtab_visits_v1 tests"

sql.diff

Click to expand! ```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql 2024-06-13 18:45:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql 2024-06-13 18:45:39.000000000 +0000 @@ -20,16 +20,6 @@ WHERE DATE(submission_timestamp) = @submission_date AND category IN ('newtab', 'topsites', 'newtab.search', 'newtab.search.ad', 'pocket') - AND name IN ( - 'closed', - 'opened', - 'impression', - 'issued', - 'click', - 'save', - 'topic_click', - 'dismiss' - ) ), visit_metadata AS ( SELECT @@ -54,6 +44,7 @@ ANY_VALUE(metrics.string.newtab_homepage_category) AS newtab_homepage_category, ANY_VALUE(metrics.string.newtab_newtab_category) AS newtab_newtab_category, ANY_VALUE(metrics.boolean.newtab_search_enabled) AS newtab_search_enabled, + ANY_VALUE(metrics.boolean.newtab_weather_enabled) AS newtab_weather_widget_enabled, ANY_VALUE(metrics.quantity.topsites_rows) AS topsites_rows, ANY_VALUE(metrics.string_list.newtab_blocked_sponsors) AS newtab_blocked_sponsors, ANY_VALUE(ping_info.experiments) AS experiments, @@ -110,6 +101,7 @@ events_unnested WHERE event_category IN ('newtab.search', 'newtab.search.ad') + AND event_name IN ('click', 'impression', 'issued') GROUP BY newtab_visit_id, search_engine, @@ -180,6 +172,7 @@ ) WHERE event_category = 'topsites' + AND event_name IN ('dismiss', 'click', 'impression') GROUP BY newtab_visit_id, topsite_tile_position, @@ -218,6 +211,8 @@ SELECT mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS pocket_story_position, + mozfun.map.get_key(event_details, "tile_id") AS pocket_tile_id, + mozfun.map.get_key(event_details, "recommendation_id") AS pocket_recommendation_id, COUNTIF(event_name = 'save') AS pocket_saves, COUNTIF(event_name = 'click') AS pocket_clicks, COUNTIF(event_name = 'impression') AS pocket_impressions, @@ -249,9 +244,12 @@ events_unnested WHERE event_category = 'pocket' + AND event_name IN ('impression', 'click', 'save') GROUP BY newtab_visit_id, - pocket_story_position + pocket_story_position, + pocket_tile_id, + pocket_recommendation_id ), pocket_summary AS ( SELECT @@ -259,6 +257,8 @@ ARRAY_AGG( STRUCT( pocket_story_position, + pocket_tile_id, + pocket_recommendation_id, pocket_impressions, sponsored_pocket_impressions, organic_pocket_impressions, @@ -275,6 +275,99 @@ GROUP BY newtab_visit_id ), +wallpaper_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + mozfun.map.get_key(event_details, "selected_wallpaper") AS wallpaper_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_click') AS wallpaper_clicks, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "true" + ) AS wallpaper_clicks_had_previous_wallpaper, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "false" + ) AS wallpaper_clicks_first_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_category_click') AS wallpaper_category_clicks, + COUNTIF(event_name = 'wallpaper_highlight_dismissed') AS wallpaper_highlight_dismissals, + COUNTIF(event_name = 'wallpaper_highlight_cta_click') AS wallpaper_highlight_cta_clicks + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'wallpaper_click', + 'wallpaper_category_click', + 'wallpaper_highlight_cta_clicks', + 'wallpaper_highlight_dismissed' + ) + GROUP BY + newtab_visit_id, + wallpaper_selected_wallpaper +), +wallpaper_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + wallpaper_selected_wallpaper, + wallpaper_clicks, + wallpaper_clicks_had_previous_wallpaper, + wallpaper_clicks_first_selected_wallpaper, + wallpaper_category_clicks, + wallpaper_highlight_dismissals, + wallpaper_highlight_cta_clicks + ) + ) AS wallpaper_interactions + FROM + wallpaper_events + GROUP BY + newtab_visit_id +), +weather_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + COUNTIF(event_name = 'weather_impression') AS weather_widget_impressions, + COUNTIF(event_name = 'weather_open_provider_url') AS weather_widget_clicks, + COUNTIF(event_name = 'weather_load_error') AS weather_widget_load_errors, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "detailed" + ) AS weather_widget_change_display_to_detailed, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "simple" + ) AS weather_widget_change_display_to_simple + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'weather_impression', + 'weather_open_provider_url', + 'weather_load_error', + 'weather_change_display' + ) + GROUP BY + newtab_visit_id +), +weather_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + weather_widget_impressions, + weather_widget_clicks, + weather_widget_load_errors, + weather_widget_change_display_to_detailed, + weather_widget_change_display_to_simple + ) + ) AS weather_interactions + FROM + weather_events + GROUP BY + newtab_visit_id +), combined_newtab_activity AS ( SELECT * @@ -289,6 +382,12 @@ LEFT JOIN pocket_summary USING (newtab_visit_id) + LEFT JOIN + wallpaper_summary + USING (newtab_visit_id) + LEFT JOIN + weather_summary + USING (newtab_visit_id) WHERE -- Keep only rows with interactions, unless we receive a valid newtab.opened event. -- This is meant to drop only interactions that only have a newtab.closed event on the same partition @@ -297,6 +396,8 @@ OR search_interactions IS NOT NULL OR topsite_tile_interactions IS NOT NULL OR pocket_interactions IS NOT NULL + OR wallpaper_interactions IS NOT NULL + OR weather_interactions IS NOT NULL ), client_profile_info AS ( SELECT @@ -311,7 +412,15 @@ client_id ) SELECT - * + *, + CASE + WHEN ( + (newtab_open_source = "about:home" AND newtab_homepage_category = "enabled") + OR (newtab_open_source = "about:newtab" AND newtab_newtab_category = "enabled") + ) + THEN "default" + ELSE "non-default" + END AS newtab_default_ui, FROM combined_newtab_activity LEFT JOIN diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml 2024-06-13 18:45:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml 2024-06-13 18:45:39.000000000 +0000 @@ -215,6 +215,12 @@ - mode: NULLABLE name: organic_pocket_saves type: INTEGER + - name: pocket_tile_id + type: STRING + mode: NULLABLE + - name: pocket_recommendation_id + type: STRING + mode: NULLABLE mode: REPEATED name: pocket_interactions type: RECORD @@ -224,3 +230,53 @@ - mode: NULLABLE name: activity_segment type: STRING +- mode: NULLABLE + name: newtab_default_ui + type: STRING +- name: newtab_weather_widget_enabled + type: BOOLEAN + mode: NULLABLE +- name: wallpaper_interactions + type: RECORD + mode: REPEATED + fields: + - name: wallpaper_selected_wallpaper + type: STRING + mode: NULLABLE + - name: wallpaper_clicks + type: INTEGER + mode: NULLABLE + - name: wallpaper_clicks_had_previous_wallpaper + type: INTEGER + mode: NULLABLE + - name: wallpaper_clicks_first_selected_wallpaper + type: INTEGER + mode: NULLABLE + - name: wallpaper_category_clicks + type: INTEGER + mode: NULLABLE + - name: wallpaper_highlight_dismissals + type: INTEGER + mode: NULLABLE + - name: wallpaper_highlight_cta_clicks + type: INTEGER + mode: NULLABLE +- name: weather_interactions + type: RECORD + mode: REPEATED + fields: + - name: weather_widget_impressions + type: INTEGER + mode: NULLABLE + - name: weather_widget_clicks + type: INTEGER + mode: NULLABLE + - name: weather_widget_load_errors + type: INTEGER + mode: NULLABLE + - name: weather_widget_change_display_to_detailed + type: INTEGER + mode: NULLABLE + - name: weather_widget_change_display_to_simple + type: INTEGER + mode: NULLABLE ```

Link to full diff

dataops-ci-bot commented 2 weeks ago

Integration report for "Fix tests again"

sql.diff

Click to expand! ```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare.py /tmp/workspace/generated-sql/dags/bqetl_cloudflare.py --- /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare.py 2024-06-13 19:13:21.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_cloudflare.py 2024-06-13 19:14:45.000000000 +0000 @@ -84,20 +84,3 @@ owner="kwindau@mozilla.com", email=["kwindau@mozilla.com"], ) - - cloudflare_derived__os_usage__v1 = GKEPodOperator( - task_id="cloudflare_derived__os_usage__v1", - arguments=[ - "python", - "sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py", - ] - + [ - "--date", - "{{ds}}", - "--cloudflare_api_token", - "{{ var.value.cloudflare_auth_token}}", - ], - image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest", - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com"], - ) Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1: query.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml 2024-06-13 19:11:30.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml 2024-06-13 19:11:31.000000000 +0000 @@ -6,15 +6,6 @@ labels: incremental: true owner1: kwindau - dag: bqetl_cloudflare -scheduling: - dag_name: bqetl_cloudflare - date_partition_parameter: dte - arguments: - - --date - - '{{ds}}' - - --cloudflare_api_token - - '{{ var.value.cloudflare_auth_token}}' bigquery: time_partitioning: type: day diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py 2024-06-13 19:09:56.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,385 +0,0 @@ -# Load libraries -import json -from datetime import datetime, timedelta -import pandas as pd -import requests -from argparse import ArgumentParser -from google.cloud import bigquery -from google.cloud import storage - -# Configurations -os_usg_configs = { - "timeout_limit": 2000, - "device_types": ["DESKTOP", "MOBILE", "OTHER", "ALL"], - "locations": [ - "ALL", - "BE", - "BG", - "CA", - "CZ", - "DE", - "DK", - "EE", - "ES", - "FI", - "FR", - "GB", - "HR", - "IE", - "IT", - "CY", - "LV", - "LT", - "LU", - "HU", - "MT", - "MX", - "NL", - "AT", - "PL", - "PT", - "RO", - "SI", - "SK", - "US", - "SE", - "GR", - ], - "bucket": "gs://moz-fx-data-prod-external-data/", - "bucket_no_gs": "moz-fx-data-prod-external-data", - "results_stg_gcs_fpth": "cloudflare/os_usage/RESULTS_STAGING/%s_results_from_%s_run.csv", - "results_archive_gcs_fpth": "cloudflare/os_usage/RESULTS_ARCHIVE/%s_results_from_%s_run.csv", - "errors_stg_gcs_fpth": "cloudflare/os_usage/ERRORS_STAGING/%s_results_from_%s_run.csv", - "errors_archive_gcs_fpth": "cloudflare/os_usage/ERRORS_ARCHIVE/%s_results_from_%s_run.csv", - "gcp_project_id": "moz-fx-data-shared-prod", - "gcp_conn_id": "google_cloud_shared_prod", - "results_bq_stg_table": "moz-fx-data-shared-prod.cloudflare_derived.os_results_stg", - "errors_bq_stg_table": "moz-fx-data-shared-prod.cloudflare_derived.os_errors_stg", -} - - -# Define a function to move a GCS object then delete the original -def move_blob(bucket_name, blob_name, destination_bucket_name, destination_blob_name): - """Moves a blob from one bucket to another with a new name.""" - storage_client = storage.Client() - source_bucket = storage_client.bucket(bucket_name) - source_blob = source_bucket.blob(blob_name) - destination_bucket = storage_client.bucket(destination_bucket_name) - destination_generation_match_precondition = 0 - - blob_copy = source_bucket.copy_blob( - source_blob, - destination_bucket, - destination_blob_name, - if_generation_match=destination_generation_match_precondition, - ) - source_bucket.delete_blob(blob_name) - - print( - "Blob {} in bucket {} moved to blob {} in bucket {}.".format( - source_blob.name, - source_bucket.name, - blob_copy.name, - destination_bucket.name, - ) - ) - - -# Function to configure the API URL -def generate_os_timeseries_api_call(strt_dt, end_dt, agg_int, location, device_type): - """Generate the API call for Operating System Usage Data.""" - if location == "ALL" and device_type == "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&format=json&aggInterval={agg_int}" - elif location != "ALL" and device_type == "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={strt_dt}T00:00:00.000Z&location={location}&format=json&aggInterval={agg_int}" - elif location == "ALL" and device_type != "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&deviceType={device_type}&format=json&aggInterval={agg_int}" - else: - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&location={location}&deviceType={device_type}&format=json&aggInterval={agg_int}" - return os_usage_api_url - - -def get_os_usage_data(date_of_interest, auth_token): - """Pull OS usage data from the Cloudflare API and save errors & results to GCS.""" - # Calculate start date and end date - logical_dag_dt = date_of_interest - logical_dag_dt_as_date = datetime.strptime(logical_dag_dt, "%Y-%m-%d").date() - start_date = logical_dag_dt_as_date - timedelta(days=4) - end_date = start_date + timedelta(days=1) - print("Start Date: ", start_date) - print("End Date: ", end_date) - - # Configure request headers - bearer_string = f"Bearer {auth_token}" - headers = {"Authorization": bearer_string} - - # Initialize the empty results & errors dataframe - result_df = pd.DataFrame( - { - "Timestamps": [], - "OS": [], - "Location": [], - "DeviceType": [], - "Share": [], - "ConfidenceLevel": [], - "AggrInterval": [], - "Normalization": [], - "LastUpdatedTS": [], - } - ) - - # Initialize an errors dataframe - errors_df = pd.DataFrame( - {"StartTime": [], "EndTime": [], "Location": [], "DeviceType": []} - ) - - # Go through all combinations, submit API requests - for device_type in os_usg_configs["device_types"]: - for loc in os_usg_configs["locations"]: - print("Device Type: ", device_type) - print("Loc: ", loc) - - # Generate the URL with given parameters - os_usage_api_url = generate_os_timeseries_api_call( - start_date, end_date, "1d", loc, device_type - ) - try: - # Call the API and save the response as JSON - response = requests.get( - os_usage_api_url, - headers=headers, - timeout=os_usg_configs["timeout_limit"], - ) - response_json = json.loads(response.text) - - # If response was successful, get the result - if response_json["success"] is True: - result = response_json["result"] - # Parse metadata - conf_lvl = result["meta"]["confidenceInfo"]["level"] - aggr_intvl = result["meta"]["aggInterval"] - nrmlztn = result["meta"]["normalization"] - lst_upd = result["meta"]["lastUpdated"] - data_dict = result["serie_0"] - - for key, val in data_dict.items(): - if key != 'timestamps': - new_result_df = pd.DataFrame( - { - "Timestamps": data_dict["timestamps"], - "OS": [key] * len(val), - "Location": [loc] * len(val), - "DeviceType": [device_type] * len(val), - "Share": val, - "ConfidenceLevel": [conf_lvl] * len(val), - "AggrInterval": [aggr_intvl] * len(val), - "Normalization": [nrmlztn] * len(val), - "LastUpdatedTS": [lst_upd] * len(val), - } - ) - result_df = pd.concat([result_df, new_result_df]) - - # If response was not successful, get the errors - else: - # errors = response_json["errors"] # Maybe add to capture, right now not using this - new_errors_df = pd.DataFrame( - { - "StartTime": [start_date], - "EndTime": [end_date], - "Location": [loc], - "DeviceType": [device_type], - } - ) - errors_df = pd.concat([errors_df, new_errors_df]) - except: - new_errors_df = pd.DataFrame( - { - "StartTime": [start_date], - "EndTime": [end_date], - "Location": [loc], - "DeviceType": [device_type], - } - ) - errors_df = pd.concat([errors_df, new_errors_df]) - - result_fpath = os_usg_configs["bucket"] + os_usg_configs["results_stg_gcs_fpth"] % ( - start_date, - logical_dag_dt, - ) - errors_fpath = os_usg_configs["bucket"] + os_usg_configs["errors_stg_gcs_fpth"] % ( - start_date, - logical_dag_dt, - ) - - result_df.to_csv(result_fpath, index=False) - errors_df.to_csv(errors_fpath, index=False) - print("Wrote errors to: ", errors_fpath) - print("Wrote results to: ", result_fpath) - - # Write a summary to the logs - len_results = str(len(result_df)) - len_errors = str(len(errors_df)) - result_summary = f"# Result Rows: {len_results}; # of Error Rows: {len_errors}" - return result_summary - - -def main(): - """Call the API, save data to GCS, load to BQ staging, delete & load to BQ gold""" - parser = ArgumentParser(description=__doc__) - parser.add_argument("--date", required=True) - parser.add_argument("--cloudflare_api_token", required=True) - parser.add_argument("--project", default="moz-fx-data-shared-prod") - parser.add_argument("--dataset", default="cloudflare_derived") - - args = parser.parse_args() - print("Running for date: ") - print(args.date) - - # STEP 1 - Pull the data from the API, save results & errors to GCS staging area - result_summary = get_os_usage_data(args.date, args.cloudflare_api_token) - print("result_summary") - print(result_summary) - - # Create a bigquery client - client = bigquery.Client(args.project) - - result_uri = os_usg_configs["bucket"] + os_usg_configs["results_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - error_uri = os_usg_configs["bucket"] + os_usg_configs["errors_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - print("result_uri") - print(result_uri) - - print("error_uri") - print(error_uri) - - # STEP 2 - Copy the result data from GCS staging to BQ staging table - load_result_csv_to_bq_stg_job = client.load_table_from_uri( - result_uri, - os_usg_configs["results_bq_stg_table"], - job_config=bigquery.LoadJobConfig( - create_disposition="CREATE_IF_NEEDED", - write_disposition="WRITE_TRUNCATE", - schema=[ - {"name": "Timestamps", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "OS", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Location", "type": "STRING", "mode": "NULLABLE"}, - {"name": "DeviceType", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Share", "type": "NUMERIC", "mode": "NULLABLE"}, - {"name": "ConfidenceLevel", "type": "STRING", "mode": "NULLABLE"}, - {"name": "AggrInterval", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Normalization", "type": "STRING", "mode": "NULLABLE"}, - {"name": "LastUpdatedTS", "type": "TIMESTAMP", "mode": "NULLABLE"}, - ], - skip_leading_rows=1, - source_format=bigquery.SourceFormat.CSV, - ), - ) - - load_result_csv_to_bq_stg_job.result() - result_bq_stg_tbl = client.get_table(os_usg_configs["results_bq_stg_table"]) - print("Loaded {} rows to results staging.".format(result_bq_stg_tbl.num_rows)) - - # STEP 3 - Copy the error data from GCS staging to BQ staging table - load_error_csv_to_bq_stg_job = client.load_table_from_uri( - error_uri, - os_usg_configs["errors_bq_stg_table"], - job_config=bigquery.LoadJobConfig( - create_disposition="CREATE_IF_NEEDED", - write_disposition="WRITE_TRUNCATE", - schema=[ - {"name": "StartTime", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "EndTime", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "Location", "type": "STRING", "mode": "NULLABLE"}, - {"name": "DeviceType", "type": "STRING", "mode": "NULLABLE"}, - ], - skip_leading_rows=1, - source_format=bigquery.SourceFormat.CSV, - ), - ) - - load_error_csv_to_bq_stg_job.result() - error_bq_stg_tbl = client.get_table(os_usg_configs["errors_bq_stg_table"]) - print("Loaded {} rows to errors staging.".format(error_bq_stg_tbl.num_rows)) - - # STEP 4 - Delete results from gold for this day, if there are any already (so if rerun, no issues will occur) - del_exstng_gold_res_for_date = f"""DELETE FROM `moz-fx-data-shared-prod.cloudflare_derived.os_usage_v1` WHERE dte = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - del_gold_res_job = client.query(del_exstng_gold_res_for_date) - del_gold_res_job.result() - print("Deleted anything already existing for this date from results gold") - - # STEP 5 - Delete errors from gold for this day, if there are any already (so if rerun, no issues will occur) - del_exstng_gold_err_for_date = f"""DELETE FROM `moz-fx-data-shared-prod.cloudflare_derived.os_usage_errors_v1` WHERE dte = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - del_gold_err_job = client.query(del_exstng_gold_err_for_date) - del_gold_err_job.result() - print("Deleted anything already existing for this date from errors gold") - - # STEP 6 - Load results from stage to gold # NEED TO UPDATE THIS STILL - device_usg_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_v1` -SELECT -CAST(Timestamps AS date) AS dte, -OS AS os, -Location AS location, -DeviceType AS device_type, -Share AS os_share, -Normalization AS normalization_type, -LastUpdatedTS AS last_updated_ts -FROM `moz-fx-data-shared-prod.cloudflare_derived.os_results_stg` -WHERE CAST(Timestamps as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - load_res_to_gold = client.query(device_usg_stg_to_gold_query) - load_res_to_gold.result() - - # STEP 7 - Load errors from stage to gold - browser_usg_errors_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_errors_v1` -SELECT -CAST(StartTime as date) AS dte, -Location AS location, -DeviceType AS device_type -FROM `moz-fx-data-shared-prod.cloudflare_derived.os_errors_stg` -WHERE CAST(StartTime as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - load_err_to_gold = client.query(browser_usg_errors_stg_to_gold_query) - load_err_to_gold.result() - - # Initialize a storage client to use in next steps - storage_client = storage.Client() - - # STEP 8 - Copy the result CSV from stage to archive, then delete from stage - # Calculate the fpaths we will use ahead of time - result_stg_fpath = os_usg_configs["results_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - result_archive_fpath = os_usg_configs["results_archive_gcs_fpath"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - move_blob( - os_usg_configs["bucket_no_gs"], - result_stg_fpath, - os_usg_configs["bucket_no_gs"], - result_archive_fpath, - ) - - # STEP 9 - Copy the error CSV from stage to archive, then delete from stage - error_stg_fpath = os_usg_configs["errors_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - error_archive_fpath = os_usg_configs["errors_archive_gcs_fpath"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - move_blob( - os_usg_configs["bucket_no_gs"], - error_stg_fpath, - os_usg_configs["bucket_no_gs"], - error_archive_fpath, - ) - - -if __name__ == "__main__": - main() diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql 2024-06-13 19:09:56.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql 2024-06-13 19:09:48.000000000 +0000 @@ -20,16 +20,6 @@ WHERE DATE(submission_timestamp) = @submission_date AND category IN ('newtab', 'topsites', 'newtab.search', 'newtab.search.ad', 'pocket') - AND name IN ( - 'closed', - 'opened', - 'impression', - 'issued', - 'click', - 'save', - 'topic_click', - 'dismiss' - ) ), visit_metadata AS ( SELECT @@ -54,6 +44,7 @@ ANY_VALUE(metrics.string.newtab_homepage_category) AS newtab_homepage_category, ANY_VALUE(metrics.string.newtab_newtab_category) AS newtab_newtab_category, ANY_VALUE(metrics.boolean.newtab_search_enabled) AS newtab_search_enabled, + ANY_VALUE(metrics.boolean.newtab_weather_enabled) AS newtab_weather_widget_enabled, ANY_VALUE(metrics.quantity.topsites_rows) AS topsites_rows, ANY_VALUE(metrics.string_list.newtab_blocked_sponsors) AS newtab_blocked_sponsors, ANY_VALUE(ping_info.experiments) AS experiments, @@ -110,6 +101,7 @@ events_unnested WHERE event_category IN ('newtab.search', 'newtab.search.ad') + AND event_name IN ('click', 'impression', 'issued') GROUP BY newtab_visit_id, search_engine, @@ -180,6 +172,7 @@ ) WHERE event_category = 'topsites' + AND event_name IN ('dismiss', 'click', 'impression') GROUP BY newtab_visit_id, topsite_tile_position, @@ -218,6 +211,8 @@ SELECT mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS pocket_story_position, + mozfun.map.get_key(event_details, "tile_id") AS pocket_tile_id, + mozfun.map.get_key(event_details, "recommendation_id") AS pocket_recommendation_id, COUNTIF(event_name = 'save') AS pocket_saves, COUNTIF(event_name = 'click') AS pocket_clicks, COUNTIF(event_name = 'impression') AS pocket_impressions, @@ -249,9 +244,12 @@ events_unnested WHERE event_category = 'pocket' + AND event_name IN ('impression', 'click', 'save') GROUP BY newtab_visit_id, - pocket_story_position + pocket_story_position, + pocket_tile_id, + pocket_recommendation_id ), pocket_summary AS ( SELECT @@ -259,6 +257,8 @@ ARRAY_AGG( STRUCT( pocket_story_position, + pocket_tile_id, + pocket_recommendation_id, pocket_impressions, sponsored_pocket_impressions, organic_pocket_impressions, @@ -275,6 +275,99 @@ GROUP BY newtab_visit_id ), +wallpaper_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + mozfun.map.get_key(event_details, "selected_wallpaper") AS wallpaper_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_click') AS wallpaper_clicks, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "true" + ) AS wallpaper_clicks_had_previous_wallpaper, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "false" + ) AS wallpaper_clicks_first_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_category_click') AS wallpaper_category_clicks, + COUNTIF(event_name = 'wallpaper_highlight_dismissed') AS wallpaper_highlight_dismissals, + COUNTIF(event_name = 'wallpaper_highlight_cta_click') AS wallpaper_highlight_cta_clicks + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'wallpaper_click', + 'wallpaper_category_click', + 'wallpaper_highlight_cta_clicks', + 'wallpaper_highlight_dismissed' + ) + GROUP BY + newtab_visit_id, + wallpaper_selected_wallpaper +), +wallpaper_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + wallpaper_selected_wallpaper, + wallpaper_clicks, + wallpaper_clicks_had_previous_wallpaper, + wallpaper_clicks_first_selected_wallpaper, + wallpaper_category_clicks, + wallpaper_highlight_dismissals, + wallpaper_highlight_cta_clicks + ) + ) AS wallpaper_interactions + FROM + wallpaper_events + GROUP BY + newtab_visit_id +), +weather_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + COUNTIF(event_name = 'weather_impression') AS weather_widget_impressions, + COUNTIF(event_name = 'weather_open_provider_url') AS weather_widget_clicks, + COUNTIF(event_name = 'weather_load_error') AS weather_widget_load_errors, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "detailed" + ) AS weather_widget_change_display_to_detailed, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "simple" + ) AS weather_widget_change_display_to_simple + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'weather_impression', + 'weather_open_provider_url', + 'weather_load_error', + 'weather_change_display' + ) + GROUP BY + newtab_visit_id +), +weather_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + weather_widget_impressions, + weather_widget_clicks, + weather_widget_load_errors, + weather_widget_change_display_to_detailed, + weather_widget_change_display_to_simple + ) + ) AS weather_interactions + FROM + weather_events + GROUP BY + newtab_visit_id +), combined_newtab_activity AS ( SELECT * @@ -289,6 +382,12 @@ LEFT JOIN pocket_summary USING (newtab_visit_id) + LEFT JOIN + wallpaper_summary + USING (newtab_visit_id) + LEFT JOIN + weather_summary + USING (newtab_visit_id) WHERE -- Keep only rows with interactions, unless we receive a valid newtab.opened event. -- This is meant to drop only interactions that only have a newtab.closed event on the same partition @@ -297,6 +396,8 @@ OR search_interactions IS NOT NULL OR topsite_tile_interactions IS NOT NULL OR pocket_interactions IS NOT NULL + OR wallpaper_interactions IS NOT NULL + OR weather_interactions IS NOT NULL ), client_profile_info AS ( SELECT @@ -311,7 +412,15 @@ client_id ) SELECT - * + *, + CASE + WHEN ( + (newtab_open_source = "about:home" AND newtab_homepage_category = "enabled") + OR (newtab_open_source = "about:newtab" AND newtab_newtab_category = "enabled") + ) + THEN "default" + ELSE "non-default" + END AS newtab_default_ui, FROM combined_newtab_activity LEFT JOIN diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml 2024-06-13 19:09:56.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml 2024-06-13 19:09:48.000000000 +0000 @@ -215,6 +215,12 @@ - mode: NULLABLE name: organic_pocket_saves type: INTEGER + - name: pocket_tile_id + type: STRING + mode: NULLABLE + - name: pocket_recommendation_id + type: STRING + mode: NULLABLE mode: REPEATED name: pocket_interactions type: RECORD @@ -224,3 +230,53 @@ - mode: NULLABLE name: activity_segment type: STRING +- mode: NULLABLE + name: newtab_default_ui + type: STRING +- name: newtab_weather_widget_enabled + type: BOOLEAN + mode: NULLABLE +- name: wallpaper_interactions + type: RECORD + mode: REPEATED + fields: + - name: wallpaper_selected_wallpaper + type: STRING + mode: NULLABLE + - name: wallpaper_clicks + type: INTEGER + mode: NULLABLE + - name: wallpaper_clicks_had_previous_wallpaper + type: INTEGER + mode: NULLABLE + - name: wallpaper_clicks_first_selected_wallpaper + type: INTEGER + mode: NULLABLE + - name: wallpaper_category_clicks + type: INTEGER + mode: NULLABLE + - name: wallpaper_highlight_dismissals + type: INTEGER + mode: NULLABLE + - name: wallpaper_highlight_cta_clicks + type: INTEGER + mode: NULLABLE +- name: weather_interactions + type: RECORD + mode: REPEATED + fields: + - name: weather_widget_impressions + type: INTEGER + mode: NULLABLE + - name: weather_widget_clicks + type: INTEGER + mode: NULLABLE + - name: weather_widget_load_errors + type: INTEGER + mode: NULLABLE + - name: weather_widget_change_display_to_detailed + type: INTEGER + mode: NULLABLE + - name: weather_widget_change_display_to_simple + type: INTEGER + mode: NULLABLE ```

Link to full diff

dataops-ci-bot commented 2 weeks ago

Integration report for "Update metadata"

sql.diff

Click to expand! ```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare.py /tmp/workspace/generated-sql/dags/bqetl_cloudflare.py --- /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare.py 2024-06-13 19:36:45.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_cloudflare.py 2024-06-13 19:52:58.000000000 +0000 @@ -84,20 +84,3 @@ owner="kwindau@mozilla.com", email=["kwindau@mozilla.com"], ) - - cloudflare_derived__os_usage__v1 = GKEPodOperator( - task_id="cloudflare_derived__os_usage__v1", - arguments=[ - "python", - "sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py", - ] - + [ - "--date", - "{{ds}}", - "--cloudflare_api_token", - "{{ var.value.cloudflare_auth_token}}", - ], - image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest", - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com"], - ) Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1: query.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:33:51.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:35:37.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.interaction_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.events_v1` + `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml 2024-06-13 19:34:53.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml 2024-06-13 19:49:51.000000000 +0000 @@ -6,15 +6,6 @@ labels: incremental: true owner1: kwindau - dag: bqetl_cloudflare -scheduling: - dag_name: bqetl_cloudflare - date_partition_parameter: dte - arguments: - - --date - - '{{ds}}' - - --cloudflare_api_token - - '{{ var.value.cloudflare_auth_token}}' bigquery: time_partitioning: type: day diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py 2024-06-13 19:33:15.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,385 +0,0 @@ -# Load libraries -import json -from datetime import datetime, timedelta -import pandas as pd -import requests -from argparse import ArgumentParser -from google.cloud import bigquery -from google.cloud import storage - -# Configurations -os_usg_configs = { - "timeout_limit": 2000, - "device_types": ["DESKTOP", "MOBILE", "OTHER", "ALL"], - "locations": [ - "ALL", - "BE", - "BG", - "CA", - "CZ", - "DE", - "DK", - "EE", - "ES", - "FI", - "FR", - "GB", - "HR", - "IE", - "IT", - "CY", - "LV", - "LT", - "LU", - "HU", - "MT", - "MX", - "NL", - "AT", - "PL", - "PT", - "RO", - "SI", - "SK", - "US", - "SE", - "GR", - ], - "bucket": "gs://moz-fx-data-prod-external-data/", - "bucket_no_gs": "moz-fx-data-prod-external-data", - "results_stg_gcs_fpth": "cloudflare/os_usage/RESULTS_STAGING/%s_results_from_%s_run.csv", - "results_archive_gcs_fpth": "cloudflare/os_usage/RESULTS_ARCHIVE/%s_results_from_%s_run.csv", - "errors_stg_gcs_fpth": "cloudflare/os_usage/ERRORS_STAGING/%s_results_from_%s_run.csv", - "errors_archive_gcs_fpth": "cloudflare/os_usage/ERRORS_ARCHIVE/%s_results_from_%s_run.csv", - "gcp_project_id": "moz-fx-data-shared-prod", - "gcp_conn_id": "google_cloud_shared_prod", - "results_bq_stg_table": "moz-fx-data-shared-prod.cloudflare_derived.os_results_stg", - "errors_bq_stg_table": "moz-fx-data-shared-prod.cloudflare_derived.os_errors_stg", -} - - -# Define a function to move a GCS object then delete the original -def move_blob(bucket_name, blob_name, destination_bucket_name, destination_blob_name): - """Moves a blob from one bucket to another with a new name.""" - storage_client = storage.Client() - source_bucket = storage_client.bucket(bucket_name) - source_blob = source_bucket.blob(blob_name) - destination_bucket = storage_client.bucket(destination_bucket_name) - destination_generation_match_precondition = 0 - - blob_copy = source_bucket.copy_blob( - source_blob, - destination_bucket, - destination_blob_name, - if_generation_match=destination_generation_match_precondition, - ) - source_bucket.delete_blob(blob_name) - - print( - "Blob {} in bucket {} moved to blob {} in bucket {}.".format( - source_blob.name, - source_bucket.name, - blob_copy.name, - destination_bucket.name, - ) - ) - - -# Function to configure the API URL -def generate_os_timeseries_api_call(strt_dt, end_dt, agg_int, location, device_type): - """Generate the API call for Operating System Usage Data.""" - if location == "ALL" and device_type == "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&format=json&aggInterval={agg_int}" - elif location != "ALL" and device_type == "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={strt_dt}T00:00:00.000Z&location={location}&format=json&aggInterval={agg_int}" - elif location == "ALL" and device_type != "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&deviceType={device_type}&format=json&aggInterval={agg_int}" - else: - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&location={location}&deviceType={device_type}&format=json&aggInterval={agg_int}" - return os_usage_api_url - - -def get_os_usage_data(date_of_interest, auth_token): - """Pull OS usage data from the Cloudflare API and save errors & results to GCS.""" - # Calculate start date and end date - logical_dag_dt = date_of_interest - logical_dag_dt_as_date = datetime.strptime(logical_dag_dt, "%Y-%m-%d").date() - start_date = logical_dag_dt_as_date - timedelta(days=4) - end_date = start_date + timedelta(days=1) - print("Start Date: ", start_date) - print("End Date: ", end_date) - - # Configure request headers - bearer_string = f"Bearer {auth_token}" - headers = {"Authorization": bearer_string} - - # Initialize the empty results & errors dataframe - result_df = pd.DataFrame( - { - "Timestamps": [], - "OS": [], - "Location": [], - "DeviceType": [], - "Share": [], - "ConfidenceLevel": [], - "AggrInterval": [], - "Normalization": [], - "LastUpdatedTS": [], - } - ) - - # Initialize an errors dataframe - errors_df = pd.DataFrame( - {"StartTime": [], "EndTime": [], "Location": [], "DeviceType": []} - ) - - # Go through all combinations, submit API requests - for device_type in os_usg_configs["device_types"]: - for loc in os_usg_configs["locations"]: - print("Device Type: ", device_type) - print("Loc: ", loc) - - # Generate the URL with given parameters - os_usage_api_url = generate_os_timeseries_api_call( - start_date, end_date, "1d", loc, device_type - ) - try: - # Call the API and save the response as JSON - response = requests.get( - os_usage_api_url, - headers=headers, - timeout=os_usg_configs["timeout_limit"], - ) - response_json = json.loads(response.text) - - # If response was successful, get the result - if response_json["success"] is True: - result = response_json["result"] - # Parse metadata - conf_lvl = result["meta"]["confidenceInfo"]["level"] - aggr_intvl = result["meta"]["aggInterval"] - nrmlztn = result["meta"]["normalization"] - lst_upd = result["meta"]["lastUpdated"] - data_dict = result["serie_0"] - - for key, val in data_dict.items(): - if key != 'timestamps': - new_result_df = pd.DataFrame( - { - "Timestamps": data_dict["timestamps"], - "OS": [key] * len(val), - "Location": [loc] * len(val), - "DeviceType": [device_type] * len(val), - "Share": val, - "ConfidenceLevel": [conf_lvl] * len(val), - "AggrInterval": [aggr_intvl] * len(val), - "Normalization": [nrmlztn] * len(val), - "LastUpdatedTS": [lst_upd] * len(val), - } - ) - result_df = pd.concat([result_df, new_result_df]) - - # If response was not successful, get the errors - else: - # errors = response_json["errors"] # Maybe add to capture, right now not using this - new_errors_df = pd.DataFrame( - { - "StartTime": [start_date], - "EndTime": [end_date], - "Location": [loc], - "DeviceType": [device_type], - } - ) - errors_df = pd.concat([errors_df, new_errors_df]) - except: - new_errors_df = pd.DataFrame( - { - "StartTime": [start_date], - "EndTime": [end_date], - "Location": [loc], - "DeviceType": [device_type], - } - ) - errors_df = pd.concat([errors_df, new_errors_df]) - - result_fpath = os_usg_configs["bucket"] + os_usg_configs["results_stg_gcs_fpth"] % ( - start_date, - logical_dag_dt, - ) - errors_fpath = os_usg_configs["bucket"] + os_usg_configs["errors_stg_gcs_fpth"] % ( - start_date, - logical_dag_dt, - ) - - result_df.to_csv(result_fpath, index=False) - errors_df.to_csv(errors_fpath, index=False) - print("Wrote errors to: ", errors_fpath) - print("Wrote results to: ", result_fpath) - - # Write a summary to the logs - len_results = str(len(result_df)) - len_errors = str(len(errors_df)) - result_summary = f"# Result Rows: {len_results}; # of Error Rows: {len_errors}" - return result_summary - - -def main(): - """Call the API, save data to GCS, load to BQ staging, delete & load to BQ gold""" - parser = ArgumentParser(description=__doc__) - parser.add_argument("--date", required=True) - parser.add_argument("--cloudflare_api_token", required=True) - parser.add_argument("--project", default="moz-fx-data-shared-prod") - parser.add_argument("--dataset", default="cloudflare_derived") - - args = parser.parse_args() - print("Running for date: ") - print(args.date) - - # STEP 1 - Pull the data from the API, save results & errors to GCS staging area - result_summary = get_os_usage_data(args.date, args.cloudflare_api_token) - print("result_summary") - print(result_summary) - - # Create a bigquery client - client = bigquery.Client(args.project) - - result_uri = os_usg_configs["bucket"] + os_usg_configs["results_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - error_uri = os_usg_configs["bucket"] + os_usg_configs["errors_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - print("result_uri") - print(result_uri) - - print("error_uri") - print(error_uri) - - # STEP 2 - Copy the result data from GCS staging to BQ staging table - load_result_csv_to_bq_stg_job = client.load_table_from_uri( - result_uri, - os_usg_configs["results_bq_stg_table"], - job_config=bigquery.LoadJobConfig( - create_disposition="CREATE_IF_NEEDED", - write_disposition="WRITE_TRUNCATE", - schema=[ - {"name": "Timestamps", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "OS", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Location", "type": "STRING", "mode": "NULLABLE"}, - {"name": "DeviceType", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Share", "type": "NUMERIC", "mode": "NULLABLE"}, - {"name": "ConfidenceLevel", "type": "STRING", "mode": "NULLABLE"}, - {"name": "AggrInterval", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Normalization", "type": "STRING", "mode": "NULLABLE"}, - {"name": "LastUpdatedTS", "type": "TIMESTAMP", "mode": "NULLABLE"}, - ], - skip_leading_rows=1, - source_format=bigquery.SourceFormat.CSV, - ), - ) - - load_result_csv_to_bq_stg_job.result() - result_bq_stg_tbl = client.get_table(os_usg_configs["results_bq_stg_table"]) - print("Loaded {} rows to results staging.".format(result_bq_stg_tbl.num_rows)) - - # STEP 3 - Copy the error data from GCS staging to BQ staging table - load_error_csv_to_bq_stg_job = client.load_table_from_uri( - error_uri, - os_usg_configs["errors_bq_stg_table"], - job_config=bigquery.LoadJobConfig( - create_disposition="CREATE_IF_NEEDED", - write_disposition="WRITE_TRUNCATE", - schema=[ - {"name": "StartTime", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "EndTime", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "Location", "type": "STRING", "mode": "NULLABLE"}, - {"name": "DeviceType", "type": "STRING", "mode": "NULLABLE"}, - ], - skip_leading_rows=1, - source_format=bigquery.SourceFormat.CSV, - ), - ) - - load_error_csv_to_bq_stg_job.result() - error_bq_stg_tbl = client.get_table(os_usg_configs["errors_bq_stg_table"]) - print("Loaded {} rows to errors staging.".format(error_bq_stg_tbl.num_rows)) - - # STEP 4 - Delete results from gold for this day, if there are any already (so if rerun, no issues will occur) - del_exstng_gold_res_for_date = f"""DELETE FROM `moz-fx-data-shared-prod.cloudflare_derived.os_usage_v1` WHERE dte = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - del_gold_res_job = client.query(del_exstng_gold_res_for_date) - del_gold_res_job.result() - print("Deleted anything already existing for this date from results gold") - - # STEP 5 - Delete errors from gold for this day, if there are any already (so if rerun, no issues will occur) - del_exstng_gold_err_for_date = f"""DELETE FROM `moz-fx-data-shared-prod.cloudflare_derived.os_usage_errors_v1` WHERE dte = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - del_gold_err_job = client.query(del_exstng_gold_err_for_date) - del_gold_err_job.result() - print("Deleted anything already existing for this date from errors gold") - - # STEP 6 - Load results from stage to gold # NEED TO UPDATE THIS STILL - device_usg_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_v1` -SELECT -CAST(Timestamps AS date) AS dte, -OS AS os, -Location AS location, -DeviceType AS device_type, -Share AS os_share, -Normalization AS normalization_type, -LastUpdatedTS AS last_updated_ts -FROM `moz-fx-data-shared-prod.cloudflare_derived.os_results_stg` -WHERE CAST(Timestamps as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - load_res_to_gold = client.query(device_usg_stg_to_gold_query) - load_res_to_gold.result() - - # STEP 7 - Load errors from stage to gold - browser_usg_errors_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_errors_v1` -SELECT -CAST(StartTime as date) AS dte, -Location AS location, -DeviceType AS device_type -FROM `moz-fx-data-shared-prod.cloudflare_derived.os_errors_stg` -WHERE CAST(StartTime as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - load_err_to_gold = client.query(browser_usg_errors_stg_to_gold_query) - load_err_to_gold.result() - - # Initialize a storage client to use in next steps - storage_client = storage.Client() - - # STEP 8 - Copy the result CSV from stage to archive, then delete from stage - # Calculate the fpaths we will use ahead of time - result_stg_fpath = os_usg_configs["results_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - result_archive_fpath = os_usg_configs["results_archive_gcs_fpath"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - move_blob( - os_usg_configs["bucket_no_gs"], - result_stg_fpath, - os_usg_configs["bucket_no_gs"], - result_archive_fpath, - ) - - # STEP 9 - Copy the error CSV from stage to archive, then delete from stage - error_stg_fpath = os_usg_configs["errors_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - error_archive_fpath = os_usg_configs["errors_archive_gcs_fpath"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - move_blob( - os_usg_configs["bucket_no_gs"], - error_stg_fpath, - os_usg_configs["bucket_no_gs"], - error_archive_fpath, - ) - - -if __name__ == "__main__": - main() diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-13 19:33:14.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-13 19:42:41.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: source + mode: NULLABLE +- name: source type: STRING -- mode: NULLABLE - name: event_type + mode: NULLABLE +- name: event_type type: STRING -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: subdivision1 + mode: NULLABLE +- name: subdivision1 type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: event_count + mode: NULLABLE +- name: event_count type: INTEGER -- mode: NULLABLE - name: user_count + mode: NULLABLE +- name: user_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-13 19:33:14.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-13 19:42:41.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: impression_count + mode: NULLABLE +- name: impression_count type: INTEGER -- mode: NULLABLE - name: click_count + mode: NULLABLE +- name: click_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-13 19:33:14.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-13 19:41:30.000000000 +0000 @@ -26,6 +26,9 @@ - name: adjust_network type: STRING mode: NULLABLE +- name: install_source + type: STRING + mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-13 19:33:14.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-13 19:41:31.000000000 +0000 @@ -48,6 +48,10 @@ description: 'The type of source of a client installation. ' +- name: install_source + type: STRING + mode: NULLABLE + description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:33:51.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:35:37.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:33:51.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:35:37.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -80,7 +80,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_android/use_counters/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/klar_android/use_counters/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_android/use_counters/schema.yaml 2024-06-13 19:33:52.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/klar_android/use_counters/schema.yaml 2024-06-13 19:46:35.000000000 +0000 @@ -1,11 +1,11 @@ fields: - name: normalized_app_id - mode: NULLABLE type: STRING + mode: NULLABLE description: App ID of the channel data was received from - name: normalized_channel - mode: NULLABLE type: STRING + mode: NULLABLE description: Normalized channel name - name: additional_properties type: STRING diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-13 19:33:51.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-13 19:37:17.000000000 +0000 @@ -45,7 +45,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -75,7 +75,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -572,7 +572,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -592,7 +592,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -663,7 +663,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -683,7 +683,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -754,7 +754,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -774,7 +774,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1565,7 +1565,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1575,7 +1575,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1656,7 +1656,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1666,7 +1666,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1747,7 +1747,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1757,7 +1757,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1838,7 +1838,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1848,7 +1848,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1990,7 +1990,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -2000,7 +2000,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2010,7 +2010,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.events_v1` + `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2162,7 +2162,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -2172,7 +2172,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:33:51.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:35:37.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.main_v1` + `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-13 19:33:14.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-13 19:42:53.000000000 +0000 @@ -1,7 +1,13 @@ fields: -- type: DATETIME - name: build_hour -- type: INTEGER - name: geckoview_major_version -- type: INTEGER - name: n_pings +- name: build_hour + type: DATETIME + mode: NULLABLE + description: null +- name: geckoview_major_version + type: INTEGER + mode: NULLABLE + description: null +- name: n_pings + type: INTEGER + mode: NULLABLE + description: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:33:51.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:35:38.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:33:51.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:35:39.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:33:52.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:35:39.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:33:51.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:35:39.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:33:52.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:35:39.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:33:51.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 19:35:39.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/clients_last_seen/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/clients_last_seen/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/clients_last_seen/schema.yaml 2024-06-13 19:33:51.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/clients_last_seen/schema.yaml 2024-06-13 19:43:07.000000000 +0000 @@ -77,44 +77,33 @@ - name: submission_date type: DATE mode: NULLABLE - description: Submission Date - name: first_seen_date type: DATE mode: NULLABLE - description: First Seen Date - name: second_seen_date type: DATE mode: NULLABLE - description: Second Seen Date - name: days_seen_bits type: INTEGER mode: NULLABLE - description: Bit pattern to identify when the client_id has sent a main ping in - the previous 28 days. - name: days_visited_1_uri_bits type: INTEGER mode: NULLABLE - description: Days Visited 1 URI Bits - name: days_visited_5_uri_bits type: INTEGER mode: NULLABLE - description: Days Visited 5 URI Bits - name: days_visited_10_uri_bits type: INTEGER mode: NULLABLE - description: Days Visited 10 URI Bits - name: days_had_8_active_ticks_bits type: INTEGER mode: NULLABLE - description: Days Had 8 Active Ticks Bits - name: days_opened_dev_tools_bits type: INTEGER mode: NULLABLE - description: Days Opened Dev Tools Bits - name: days_interacted_bits type: INTEGER mode: NULLABLE - description: Days Interacted Bits - name: days_visited_1_uri_normal_mode_bits type: INTEGER mode: NULLABLE @@ -143,23 +132,18 @@ - name: client_id type: STRING mode: NULLABLE - description: Client ID - name: aborts_content_sum type: INTEGER mode: NULLABLE - description: Aborts Content Sum - name: aborts_gmplugin_sum type: INTEGER mode: NULLABLE - description: null - name: aborts_plugin_sum type: INTEGER mode: NULLABLE - description: Aborts Plugin Sum - name: active_addons_count_mean type: FLOAT mode: NULLABLE - description: Active Addons Count Mean - name: active_addons type: RECORD mode: REPEATED @@ -167,92 +151,69 @@ - name: addon_id type: STRING mode: NULLABLE - description: Add-On Identifier - name: blocklisted type: BOOLEAN mode: NULLABLE - description: Blocklisted - name: name type: STRING mode: NULLABLE - description: Name - name: user_disabled type: BOOLEAN mode: NULLABLE - description: User Disabled - name: app_disabled type: BOOLEAN mode: NULLABLE - description: App Disabled - name: version type: STRING mode: NULLABLE - description: Version - name: scope type: INTEGER mode: NULLABLE - description: Scope - name: type type: STRING mode: NULLABLE - description: Type - name: foreign_install type: BOOLEAN mode: NULLABLE - description: Foreign Install - name: has_binary_components type: BOOLEAN mode: NULLABLE - description: Has Binary Components - name: install_day type: INTEGER mode: NULLABLE - description: Install Day - name: update_day type: INTEGER mode: NULLABLE - description: Update Day - name: signed_state type: INTEGER mode: NULLABLE - description: Signed State - name: is_system type: BOOLEAN mode: NULLABLE - description: Is System - name: is_web_extension type: BOOLEAN mode: NULLABLE - description: Is Web Extension - name: multiprocess_compatible type: BOOLEAN mode: NULLABLE - description: Multiprocess Compatible - description: Active Addons - name: active_hours_sum type: FLOAT mode: NULLABLE - description: Active Hours Sum - name: addon_compatibility_check_enabled type: BOOLEAN mode: NULLABLE - description: Addon Compatibility Check Enabled - name: app_build_id type: STRING mode: NULLABLE - description: App Build ID - name: app_display_version type: STRING mode: NULLABLE - description: App Display Version - name: app_name type: STRING mode: NULLABLE - description: App Name - name: app_version type: STRING mode: NULLABLE - description: App Version - name: attribution type: RECORD mode: NULLABLE @@ -260,196 +221,147 @@ - name: source type: STRING mode: NULLABLE - description: Source - name: medium type: STRING mode: NULLABLE - description: Medium - name: campaign type: STRING mode: NULLABLE - description: Campaign - name: content type: STRING mode: NULLABLE - description: Content - name: experiment type: STRING mode: NULLABLE - description: Experiment - name: variation type: STRING mode: NULLABLE - description: Variation - name: dltoken type: STRING mode: NULLABLE - description: Download Token - name: dlsource type: STRING mode: NULLABLE - description: Download Source - name: ua type: STRING mode: NULLABLE - description: null - description: Attribution - name: blocklist_enabled type: BOOLEAN mode: NULLABLE - description: Blocklist Enabled - name: channel type: STRING mode: NULLABLE - description: Channel - name: client_clock_skew_mean type: FLOAT mode: NULLABLE - description: Client Clock Skew Mean - name: client_submission_latency_mean type: FLOAT mode: NULLABLE - description: Client Submission Latency Mean - name: cpu_cores type: INTEGER mode: NULLABLE - description: CPU Cores - name: cpu_count type: INTEGER mode: NULLABLE - description: CPU Count - name: cpu_family type: INTEGER mode: NULLABLE - description: CPU Family - name: cpu_l2_cache_kb type: INTEGER mode: NULLABLE - description: CPU L2 Cache KB - name: cpu_l3_cache_kb type: INTEGER mode: NULLABLE - description: CPU L3 Cache KB - name: cpu_model type: INTEGER mode: NULLABLE - description: CPU Model - name: cpu_speed_mhz type: INTEGER mode: NULLABLE - description: CPU Speed MHz - name: cpu_stepping type: INTEGER mode: NULLABLE - description: CPU Stepping - name: cpu_vendor type: STRING mode: NULLABLE - description: CPU Vendor - name: crashes_detected_content_sum type: INTEGER mode: NULLABLE - description: Crashes Detected Content Sum - name: crashes_detected_gmplugin_sum type: INTEGER mode: NULLABLE - description: null - name: crashes_detected_plugin_sum type: INTEGER mode: NULLABLE - description: Crashes Detected Plugin Sum - name: crash_submit_attempt_content_sum type: INTEGER mode: NULLABLE - description: Crash Submit Attempt Content Sum - name: crash_submit_attempt_main_sum type: INTEGER mode: NULLABLE - description: Crash Submit Attempt Main Sum - name: crash_submit_attempt_plugin_sum type: INTEGER mode: NULLABLE - description: Crash Submit Attempt Plugin Sum - name: crash_submit_success_content_sum type: INTEGER mode: NULLABLE - description: Crash Submit Success Content Sum - name: crash_submit_success_main_sum type: INTEGER mode: NULLABLE - description: Crash Submit Success Main Sum - name: crash_submit_success_plugin_sum type: INTEGER mode: NULLABLE - description: Crash Submit Success Plugin Sum - name: default_search_engine type: STRING mode: NULLABLE - description: Default Search Engine - name: default_search_engine_data_load_path type: STRING mode: NULLABLE - description: Default Search Engine Data Load Path - name: default_search_engine_data_name type: STRING mode: NULLABLE - description: Default Search Engine Data Name - name: default_search_engine_data_origin type: STRING mode: NULLABLE - description: Default Search Engine Data origin - name: default_search_engine_data_submission_url type: STRING mode: NULLABLE - description: Default Search Engine Data Submission URL - name: devtools_toolbox_opened_count_sum type: INTEGER mode: NULLABLE - description: Dev Tools Toolbox Opened Count Sum - name: distribution_id type: STRING mode: NULLABLE - description: Distribution ID - name: e10s_enabled type: BOOLEAN mode: NULLABLE - description: E10s Enabled - name: env_build_arch type: STRING mode: NULLABLE - description: Environment Build Arch - name: env_build_id type: STRING mode: NULLABLE - description: Environment Build ID - name: env_build_version type: STRING mode: NULLABLE - description: Environment Build Version - name: environment_settings_intl_accept_languages type: STRING mode: REPEATED - description: Environment Settings Intl Accept Languages - name: environment_settings_intl_app_locales type: STRING mode: REPEATED - description: Environment Settings Intl App Locales - name: environment_settings_intl_available_locales type: STRING mode: REPEATED - description: Environment Settings Intl Available Locales - name: environment_settings_intl_requested_locales type: STRING mode: REPEATED - description: Environment Settings Intl Requested Localed - name: environment_settings_intl_system_locales type: STRING mode: REPEATED - description: Environment Settings Intl System Locales - name: environment_settings_intl_regional_prefs_locales type: STRING mode: REPEATED - description: Environment Settings Intl Regional Prefs Locales - name: experiments type: RECORD mode: REPEATED @@ -457,351 +369,264 @@ - name: key type: STRING mode: NULLABLE - description: Key - name: value type: STRING mode: NULLABLE - description: Value - description: Experiments - name: first_paint_mean type: FLOAT mode: NULLABLE - description: First Paint Mean - name: flash_version type: STRING mode: NULLABLE - description: Flash Version - name: country type: STRING mode: NULLABLE - description: Country - name: city type: STRING mode: NULLABLE - description: City - name: geo_subdivision1 type: STRING mode: NULLABLE - description: Geo Subdivision 1 - name: geo_subdivision2 type: STRING mode: NULLABLE - description: Geo Subdivision 2 - name: isp_name type: STRING mode: NULLABLE - description: ISP Name - name: isp_organization type: STRING mode: NULLABLE - description: ISP Organization - name: gfx_features_advanced_layers_status type: STRING mode: NULLABLE - description: GFX Features Advanced Layers Status - name: gfx_features_d2d_status type: STRING mode: NULLABLE - description: GFX Features D2D Status - name: gfx_features_d3d11_status type: STRING mode: NULLABLE - description: GFX Features D3D11 Status - name: gfx_features_gpu_process_status type: STRING mode: NULLABLE - description: GFX Features GPU Process Status - name: histogram_parent_devtools_aboutdebugging_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools About Debugging Opened Count Sum - name: histogram_parent_devtools_animationinspector_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Animation Inspector Opened Count Sum - name: histogram_parent_devtools_browserconsole_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Browser Console Opened Count Sum - name: histogram_parent_devtools_canvasdebugger_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Canvas Debugger Opened Count Sum - name: histogram_parent_devtools_computedview_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Computed View Opened Count Sum - name: histogram_parent_devtools_custom_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Custom Opened Count Sum - name: histogram_parent_devtools_dom_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools DOM Opened Count Sum - name: histogram_parent_devtools_eyedropper_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Eyedropper Opened Count Sum - name: histogram_parent_devtools_fontinspector_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Font Inspector Opened Count Sum - name: histogram_parent_devtools_inspector_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Inspector Opened Count Sum - name: histogram_parent_devtools_jsbrowserdebugger_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools JS Browser Debugger Opened Count Sum - name: histogram_parent_devtools_jsdebugger_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Devtools JS Debugger Opened Count Sum - name: histogram_parent_devtools_jsprofiler_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools JS Profiler Opened Count Sum - name: histogram_parent_devtools_layoutview_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_memory_opened_count_sum type: INTEGER mode: NULLABLE - name: histogram_parent_devtools_menu_eyedropper_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_netmonitor_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_options_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_paintflashing_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_picker_eyedropper_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_responsive_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Responsive Opened Count Sum - name: histogram_parent_devtools_ruleview_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Rule View Opened Count Sum - name: histogram_parent_devtools_scratchpad_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Scratchpad Opened Count Sum - name: histogram_parent_devtools_scratchpad_window_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Scratchpad Window Opened Count Sum - name: histogram_parent_devtools_shadereditor_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Shader Editor Opened Count Sum - name: histogram_parent_devtools_storage_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Storage Opened Count Sum - name: histogram_parent_devtools_styleeditor_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Style Editor Opened Count Sum - name: histogram_parent_devtools_webaudioeditor_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Web Audio Editor Opened Count Sum - name: histogram_parent_devtools_webconsole_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_webide_opened_count_sum type: INTEGER mode: NULLABLE - description: Histogram Parent Dev Tools Web IDE Opened Count Sum - name: install_year type: INTEGER mode: NULLABLE - description: Install Year - name: is_default_browser type: BOOLEAN mode: NULLABLE - description: Is Default Browser Indicator - name: is_wow64 type: BOOLEAN mode: NULLABLE - description: WoW64 Indicator - name: locale type: STRING mode: NULLABLE - description: Locale - name: memory_mb type: INTEGER mode: NULLABLE - description: Memory (MB) - name: normalized_channel type: STRING mode: NULLABLE - description: Normalized Channel - name: normalized_os_version type: STRING mode: NULLABLE - description: Normalized OS Version - name: os type: STRING mode: NULLABLE - description: Operating System - name: os_service_pack_major type: INTEGER mode: NULLABLE - description: OS Service Pack Major - name: os_service_pack_minor type: INTEGER mode: NULLABLE - description: OS Service Pack Minor - name: os_version type: STRING mode: NULLABLE - description: Operating System Version - name: pings_aggregated_by_this_row type: INTEGER mode: NULLABLE - description: Pings Aggregated By This Row - name: places_bookmarks_count_mean type: FLOAT mode: NULLABLE - description: Places Bookmarks Count Mean - name: places_pages_count_mean type: FLOAT mode: NULLABLE - description: Places Pages Count Mean - name: plugin_hangs_sum type: INTEGER mode: NULLABLE - description: Plugin Hangs Sum - name: plugins_infobar_allow_sum type: INTEGER mode: NULLABLE - description: Plugins Infobar Allow Sum - name: plugins_infobar_block_sum type: INTEGER mode: NULLABLE - description: Plugins Infobar Block Sum - name: plugins_infobar_shown_sum type: INTEGER mode: NULLABLE - description: Plugins Infobar Shown Sum - name: p ```

⚠️ Only part of the diff is displayed.

Link to full diff

dataops-ci-bot commented 2 weeks ago

Integration report for "Add new boolean to test JSON"

sql.diff

Click to expand! ```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare.py /tmp/workspace/generated-sql/dags/bqetl_cloudflare.py --- /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare.py 2024-06-13 20:10:09.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_cloudflare.py 2024-06-13 20:27:16.000000000 +0000 @@ -84,20 +84,3 @@ owner="kwindau@mozilla.com", email=["kwindau@mozilla.com"], ) - - cloudflare_derived__os_usage__v1 = GKEPodOperator( - task_id="cloudflare_derived__os_usage__v1", - arguments=[ - "python", - "sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py", - ] - + [ - "--date", - "{{ds}}", - "--cloudflare_api_token", - "{{ var.value.cloudflare_auth_token}}", - ], - image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest", - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com"], - ) Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1: query.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:07:13.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:09:07.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.interaction_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml 2024-06-13 20:08:11.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml 2024-06-13 20:24:12.000000000 +0000 @@ -6,15 +6,6 @@ labels: incremental: true owner1: kwindau - dag: bqetl_cloudflare -scheduling: - dag_name: bqetl_cloudflare - date_partition_parameter: dte - arguments: - - --date - - '{{ds}}' - - --cloudflare_api_token - - '{{ var.value.cloudflare_auth_token}}' bigquery: time_partitioning: type: day diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py 2024-06-13 20:06:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,385 +0,0 @@ -# Load libraries -import json -from datetime import datetime, timedelta -import pandas as pd -import requests -from argparse import ArgumentParser -from google.cloud import bigquery -from google.cloud import storage - -# Configurations -os_usg_configs = { - "timeout_limit": 2000, - "device_types": ["DESKTOP", "MOBILE", "OTHER", "ALL"], - "locations": [ - "ALL", - "BE", - "BG", - "CA", - "CZ", - "DE", - "DK", - "EE", - "ES", - "FI", - "FR", - "GB", - "HR", - "IE", - "IT", - "CY", - "LV", - "LT", - "LU", - "HU", - "MT", - "MX", - "NL", - "AT", - "PL", - "PT", - "RO", - "SI", - "SK", - "US", - "SE", - "GR", - ], - "bucket": "gs://moz-fx-data-prod-external-data/", - "bucket_no_gs": "moz-fx-data-prod-external-data", - "results_stg_gcs_fpth": "cloudflare/os_usage/RESULTS_STAGING/%s_results_from_%s_run.csv", - "results_archive_gcs_fpth": "cloudflare/os_usage/RESULTS_ARCHIVE/%s_results_from_%s_run.csv", - "errors_stg_gcs_fpth": "cloudflare/os_usage/ERRORS_STAGING/%s_results_from_%s_run.csv", - "errors_archive_gcs_fpth": "cloudflare/os_usage/ERRORS_ARCHIVE/%s_results_from_%s_run.csv", - "gcp_project_id": "moz-fx-data-shared-prod", - "gcp_conn_id": "google_cloud_shared_prod", - "results_bq_stg_table": "moz-fx-data-shared-prod.cloudflare_derived.os_results_stg", - "errors_bq_stg_table": "moz-fx-data-shared-prod.cloudflare_derived.os_errors_stg", -} - - -# Define a function to move a GCS object then delete the original -def move_blob(bucket_name, blob_name, destination_bucket_name, destination_blob_name): - """Moves a blob from one bucket to another with a new name.""" - storage_client = storage.Client() - source_bucket = storage_client.bucket(bucket_name) - source_blob = source_bucket.blob(blob_name) - destination_bucket = storage_client.bucket(destination_bucket_name) - destination_generation_match_precondition = 0 - - blob_copy = source_bucket.copy_blob( - source_blob, - destination_bucket, - destination_blob_name, - if_generation_match=destination_generation_match_precondition, - ) - source_bucket.delete_blob(blob_name) - - print( - "Blob {} in bucket {} moved to blob {} in bucket {}.".format( - source_blob.name, - source_bucket.name, - blob_copy.name, - destination_bucket.name, - ) - ) - - -# Function to configure the API URL -def generate_os_timeseries_api_call(strt_dt, end_dt, agg_int, location, device_type): - """Generate the API call for Operating System Usage Data.""" - if location == "ALL" and device_type == "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&format=json&aggInterval={agg_int}" - elif location != "ALL" and device_type == "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={strt_dt}T00:00:00.000Z&location={location}&format=json&aggInterval={agg_int}" - elif location == "ALL" and device_type != "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&deviceType={device_type}&format=json&aggInterval={agg_int}" - else: - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&location={location}&deviceType={device_type}&format=json&aggInterval={agg_int}" - return os_usage_api_url - - -def get_os_usage_data(date_of_interest, auth_token): - """Pull OS usage data from the Cloudflare API and save errors & results to GCS.""" - # Calculate start date and end date - logical_dag_dt = date_of_interest - logical_dag_dt_as_date = datetime.strptime(logical_dag_dt, "%Y-%m-%d").date() - start_date = logical_dag_dt_as_date - timedelta(days=4) - end_date = start_date + timedelta(days=1) - print("Start Date: ", start_date) - print("End Date: ", end_date) - - # Configure request headers - bearer_string = f"Bearer {auth_token}" - headers = {"Authorization": bearer_string} - - # Initialize the empty results & errors dataframe - result_df = pd.DataFrame( - { - "Timestamps": [], - "OS": [], - "Location": [], - "DeviceType": [], - "Share": [], - "ConfidenceLevel": [], - "AggrInterval": [], - "Normalization": [], - "LastUpdatedTS": [], - } - ) - - # Initialize an errors dataframe - errors_df = pd.DataFrame( - {"StartTime": [], "EndTime": [], "Location": [], "DeviceType": []} - ) - - # Go through all combinations, submit API requests - for device_type in os_usg_configs["device_types"]: - for loc in os_usg_configs["locations"]: - print("Device Type: ", device_type) - print("Loc: ", loc) - - # Generate the URL with given parameters - os_usage_api_url = generate_os_timeseries_api_call( - start_date, end_date, "1d", loc, device_type - ) - try: - # Call the API and save the response as JSON - response = requests.get( - os_usage_api_url, - headers=headers, - timeout=os_usg_configs["timeout_limit"], - ) - response_json = json.loads(response.text) - - # If response was successful, get the result - if response_json["success"] is True: - result = response_json["result"] - # Parse metadata - conf_lvl = result["meta"]["confidenceInfo"]["level"] - aggr_intvl = result["meta"]["aggInterval"] - nrmlztn = result["meta"]["normalization"] - lst_upd = result["meta"]["lastUpdated"] - data_dict = result["serie_0"] - - for key, val in data_dict.items(): - if key != 'timestamps': - new_result_df = pd.DataFrame( - { - "Timestamps": data_dict["timestamps"], - "OS": [key] * len(val), - "Location": [loc] * len(val), - "DeviceType": [device_type] * len(val), - "Share": val, - "ConfidenceLevel": [conf_lvl] * len(val), - "AggrInterval": [aggr_intvl] * len(val), - "Normalization": [nrmlztn] * len(val), - "LastUpdatedTS": [lst_upd] * len(val), - } - ) - result_df = pd.concat([result_df, new_result_df]) - - # If response was not successful, get the errors - else: - # errors = response_json["errors"] # Maybe add to capture, right now not using this - new_errors_df = pd.DataFrame( - { - "StartTime": [start_date], - "EndTime": [end_date], - "Location": [loc], - "DeviceType": [device_type], - } - ) - errors_df = pd.concat([errors_df, new_errors_df]) - except: - new_errors_df = pd.DataFrame( - { - "StartTime": [start_date], - "EndTime": [end_date], - "Location": [loc], - "DeviceType": [device_type], - } - ) - errors_df = pd.concat([errors_df, new_errors_df]) - - result_fpath = os_usg_configs["bucket"] + os_usg_configs["results_stg_gcs_fpth"] % ( - start_date, - logical_dag_dt, - ) - errors_fpath = os_usg_configs["bucket"] + os_usg_configs["errors_stg_gcs_fpth"] % ( - start_date, - logical_dag_dt, - ) - - result_df.to_csv(result_fpath, index=False) - errors_df.to_csv(errors_fpath, index=False) - print("Wrote errors to: ", errors_fpath) - print("Wrote results to: ", result_fpath) - - # Write a summary to the logs - len_results = str(len(result_df)) - len_errors = str(len(errors_df)) - result_summary = f"# Result Rows: {len_results}; # of Error Rows: {len_errors}" - return result_summary - - -def main(): - """Call the API, save data to GCS, load to BQ staging, delete & load to BQ gold""" - parser = ArgumentParser(description=__doc__) - parser.add_argument("--date", required=True) - parser.add_argument("--cloudflare_api_token", required=True) - parser.add_argument("--project", default="moz-fx-data-shared-prod") - parser.add_argument("--dataset", default="cloudflare_derived") - - args = parser.parse_args() - print("Running for date: ") - print(args.date) - - # STEP 1 - Pull the data from the API, save results & errors to GCS staging area - result_summary = get_os_usage_data(args.date, args.cloudflare_api_token) - print("result_summary") - print(result_summary) - - # Create a bigquery client - client = bigquery.Client(args.project) - - result_uri = os_usg_configs["bucket"] + os_usg_configs["results_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - error_uri = os_usg_configs["bucket"] + os_usg_configs["errors_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - print("result_uri") - print(result_uri) - - print("error_uri") - print(error_uri) - - # STEP 2 - Copy the result data from GCS staging to BQ staging table - load_result_csv_to_bq_stg_job = client.load_table_from_uri( - result_uri, - os_usg_configs["results_bq_stg_table"], - job_config=bigquery.LoadJobConfig( - create_disposition="CREATE_IF_NEEDED", - write_disposition="WRITE_TRUNCATE", - schema=[ - {"name": "Timestamps", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "OS", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Location", "type": "STRING", "mode": "NULLABLE"}, - {"name": "DeviceType", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Share", "type": "NUMERIC", "mode": "NULLABLE"}, - {"name": "ConfidenceLevel", "type": "STRING", "mode": "NULLABLE"}, - {"name": "AggrInterval", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Normalization", "type": "STRING", "mode": "NULLABLE"}, - {"name": "LastUpdatedTS", "type": "TIMESTAMP", "mode": "NULLABLE"}, - ], - skip_leading_rows=1, - source_format=bigquery.SourceFormat.CSV, - ), - ) - - load_result_csv_to_bq_stg_job.result() - result_bq_stg_tbl = client.get_table(os_usg_configs["results_bq_stg_table"]) - print("Loaded {} rows to results staging.".format(result_bq_stg_tbl.num_rows)) - - # STEP 3 - Copy the error data from GCS staging to BQ staging table - load_error_csv_to_bq_stg_job = client.load_table_from_uri( - error_uri, - os_usg_configs["errors_bq_stg_table"], - job_config=bigquery.LoadJobConfig( - create_disposition="CREATE_IF_NEEDED", - write_disposition="WRITE_TRUNCATE", - schema=[ - {"name": "StartTime", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "EndTime", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "Location", "type": "STRING", "mode": "NULLABLE"}, - {"name": "DeviceType", "type": "STRING", "mode": "NULLABLE"}, - ], - skip_leading_rows=1, - source_format=bigquery.SourceFormat.CSV, - ), - ) - - load_error_csv_to_bq_stg_job.result() - error_bq_stg_tbl = client.get_table(os_usg_configs["errors_bq_stg_table"]) - print("Loaded {} rows to errors staging.".format(error_bq_stg_tbl.num_rows)) - - # STEP 4 - Delete results from gold for this day, if there are any already (so if rerun, no issues will occur) - del_exstng_gold_res_for_date = f"""DELETE FROM `moz-fx-data-shared-prod.cloudflare_derived.os_usage_v1` WHERE dte = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - del_gold_res_job = client.query(del_exstng_gold_res_for_date) - del_gold_res_job.result() - print("Deleted anything already existing for this date from results gold") - - # STEP 5 - Delete errors from gold for this day, if there are any already (so if rerun, no issues will occur) - del_exstng_gold_err_for_date = f"""DELETE FROM `moz-fx-data-shared-prod.cloudflare_derived.os_usage_errors_v1` WHERE dte = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - del_gold_err_job = client.query(del_exstng_gold_err_for_date) - del_gold_err_job.result() - print("Deleted anything already existing for this date from errors gold") - - # STEP 6 - Load results from stage to gold # NEED TO UPDATE THIS STILL - device_usg_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_v1` -SELECT -CAST(Timestamps AS date) AS dte, -OS AS os, -Location AS location, -DeviceType AS device_type, -Share AS os_share, -Normalization AS normalization_type, -LastUpdatedTS AS last_updated_ts -FROM `moz-fx-data-shared-prod.cloudflare_derived.os_results_stg` -WHERE CAST(Timestamps as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - load_res_to_gold = client.query(device_usg_stg_to_gold_query) - load_res_to_gold.result() - - # STEP 7 - Load errors from stage to gold - browser_usg_errors_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_errors_v1` -SELECT -CAST(StartTime as date) AS dte, -Location AS location, -DeviceType AS device_type -FROM `moz-fx-data-shared-prod.cloudflare_derived.os_errors_stg` -WHERE CAST(StartTime as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - load_err_to_gold = client.query(browser_usg_errors_stg_to_gold_query) - load_err_to_gold.result() - - # Initialize a storage client to use in next steps - storage_client = storage.Client() - - # STEP 8 - Copy the result CSV from stage to archive, then delete from stage - # Calculate the fpaths we will use ahead of time - result_stg_fpath = os_usg_configs["results_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - result_archive_fpath = os_usg_configs["results_archive_gcs_fpath"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - move_blob( - os_usg_configs["bucket_no_gs"], - result_stg_fpath, - os_usg_configs["bucket_no_gs"], - result_archive_fpath, - ) - - # STEP 9 - Copy the error CSV from stage to archive, then delete from stage - error_stg_fpath = os_usg_configs["errors_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - error_archive_fpath = os_usg_configs["errors_archive_gcs_fpath"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - move_blob( - os_usg_configs["bucket_no_gs"], - error_stg_fpath, - os_usg_configs["bucket_no_gs"], - error_archive_fpath, - ) - - -if __name__ == "__main__": - main() diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-13 20:06:36.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-13 20:14:18.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: source + mode: NULLABLE +- name: source type: STRING -- mode: NULLABLE - name: event_type + mode: NULLABLE +- name: event_type type: STRING -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: subdivision1 + mode: NULLABLE +- name: subdivision1 type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: event_count + mode: NULLABLE +- name: event_count type: INTEGER -- mode: NULLABLE - name: user_count + mode: NULLABLE +- name: user_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-13 20:06:36.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-13 20:14:17.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: impression_count + mode: NULLABLE +- name: impression_count type: INTEGER -- mode: NULLABLE - name: click_count + mode: NULLABLE +- name: click_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-13 20:06:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-13 20:15:39.000000000 +0000 @@ -26,6 +26,9 @@ - name: adjust_network type: STRING mode: NULLABLE +- name: install_source + type: STRING + mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-13 20:06:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-13 20:15:40.000000000 +0000 @@ -48,6 +48,10 @@ description: 'The type of source of a client installation. ' +- name: install_source + type: STRING + mode: NULLABLE + description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/use_counters/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/use_counters/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/use_counters/schema.yaml 2024-06-13 20:07:14.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/use_counters/schema.yaml 2024-06-13 20:17:41.000000000 +0000 @@ -1,11 +1,11 @@ fields: - name: normalized_app_id - type: STRING mode: NULLABLE + type: STRING description: App ID of the channel data was received from - name: normalized_channel - type: STRING mode: NULLABLE + type: STRING description: Normalized channel name - name: additional_properties type: STRING diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:07:14.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:09:08.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-13 20:07:14.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-13 20:10:44.000000000 +0000 @@ -45,7 +45,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -572,7 +572,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -582,7 +582,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -592,7 +592,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -663,7 +663,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -673,7 +673,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -683,7 +683,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -754,7 +754,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -764,7 +764,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -774,7 +774,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1565,7 +1565,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1575,7 +1575,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1656,7 +1656,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1666,7 +1666,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1747,7 +1747,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1757,7 +1757,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1838,7 +1838,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1848,7 +1848,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1990,7 +1990,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2000,7 +2000,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:07:13.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:09:07.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.main_v1` + `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-13 20:06:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-13 20:14:27.000000000 +0000 @@ -1,7 +1,13 @@ fields: -- type: DATETIME - name: build_hour -- type: INTEGER - name: geckoview_major_version -- type: INTEGER - name: n_pings +- name: build_hour + type: DATETIME + mode: NULLABLE + description: null +- name: geckoview_major_version + type: INTEGER + mode: NULLABLE + description: null +- name: n_pings + type: INTEGER + mode: NULLABLE + description: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:07:14.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:09:09.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:07:14.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:09:09.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:07:13.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:09:10.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:07:13.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:09:09.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:07:13.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:09:10.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:07:14.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 20:09:10.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/eng_workflow_build_parquet/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/eng_workflow_build_parquet/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/eng_workflow_build_parquet/schema.yaml 2024-06-13 20:07:15.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/eng_workflow_build_parquet/schema.yaml 2024-06-13 20:14:34.000000000 +0000 @@ -28,48 +28,36 @@ - name: artifact type: BOOLEAN mode: NULLABLE - description: true if --enable-artifact-builds - name: ccache type: BOOLEAN mode: NULLABLE - description: true if ccache is in use (--with-ccache) - name: compiler type: STRING mode: NULLABLE - description: The compiler type in use (CC_TYPE) - name: debug type: BOOLEAN mode: NULLABLE - description: true if build is debug (--enable-debug) - name: icecream type: BOOLEAN mode: NULLABLE - description: true if icecream in use - name: opt type: BOOLEAN mode: NULLABLE - description: true if build is optimized (--enable-optimize) - name: sccache type: BOOLEAN mode: NULLABLE - description: true if ccache in use is sccache - description: Selected build options - name: client_id type: STRING mode: NULLABLE - description: A UUID to uniquely identify a client - name: command type: STRING mode: NULLABLE - description: The mach command that was invoked - name: duration_ms type: FLOAT mode: NULLABLE - description: Command duration in milliseconds - name: success type: BOOLEAN mode: NULLABLE - description: true if the command succeeded - name: system type: RECORD mode: NULLABLE @@ -77,40 +65,30 @@ - name: os type: STRING mode: NULLABLE - description: Operating system - name: cpu_brand type: STRING mode: NULLABLE - description: CPU brand string from CPUID - name: drive_is_ssd type: BOOLEAN mode: NULLABLE - description: true if the source directory is on a solid-state disk - name: logical_cores type: INTEGER mode: NULLABLE - description: Number of logical CPU cores present - name: memory_gb type: INTEGER mode: NULLABLE - description: System memory in GB - name: physical_cores type: INTEGER mode: NULLABLE - description: Number of physical CPU cores present - name: virtual_machine type: BOOLEAN mode: NULLABLE - description: true if the OS appears to be running in a virtual machine - name: time type: STRING mode: NULLABLE - description: Time at which this event happened - name: exception type: STRING mode: NULLABLE - description: If a Python exception was encountered during the execution of the command, - this value contains the result of calling `repr` on the exception object. - name: file_types_changed type: RECORD mode: REPEATED @@ -118,11 +96,9 @@ - name: count type: INTEGER mode: NULLABLE - description: Count of changed files with this extension - name: ext type: STRING mode: NULLABLE - description: File extension - name: build_attrs type: RECORD mode: NULLABLE @@ -130,9 +106,6 @@ - name: clobber type: BOOLEAN mode: NULLABLE - description: true if the build was a clobber/full build - name: cpu_percent type: FLOAT mode: NULLABLE - description: cpu utilization observed during a build - description: Attributes characterizing a build diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql 2024-06-13 20:06:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql 2024-06-13 20:07:25.000000000 +0000 @@ -20,16 +20,6 @@ WHERE DATE(submission_timestamp) = @submission_date AND category IN ('newtab', 'topsites', 'newtab.search', 'newtab.search.ad', 'pocket') - AND name IN ( - 'closed', - 'opened', - 'impression', - 'issued', - 'click', - 'save', - 'topic_click', - 'dismiss' - ) ), visit_metadata AS ( SELECT @@ -54,6 +44,7 @@ ANY_VALUE(metrics.string.newtab_homepage_category) AS newtab_homepage_category, ANY_VALUE(metrics.string.newtab_newtab_category) AS newtab_newtab_category, ANY_VALUE(metrics.boolean.newtab_search_enabled) AS newtab_search_enabled, + ANY_VALUE(metrics.boolean.newtab_weather_enabled) AS newtab_weather_widget_enabled, ANY_VALUE(metrics.quantity.topsites_rows) AS topsites_rows, ANY_VALUE(metrics.string_list.newtab_blocked_sponsors) AS newtab_blocked_sponsors, ANY_VALUE(ping_info.experiments) AS experiments, @@ -110,6 +101,7 @@ events_unnested WHERE event_category IN ('newtab.search', 'newtab.search.ad') + AND event_name IN ('click', 'impression', 'issued') GROUP BY newtab_visit_id, search_engine, @@ -180,6 +172,7 @@ ) WHERE event_category = 'topsites' + AND event_name IN ('dismiss', 'click', 'impression') GROUP BY newtab_visit_id, topsite_tile_position, @@ -218,6 +211,8 @@ SELECT mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS pocket_story_position, + mozfun.map.get_key(event_details, "tile_id") AS pocket_tile_id, + mozfun.map.get_key(event_details, "recommendation_id") AS pocket_recommendation_id, COUNTIF(event_name = 'save') AS pocket_saves, COUNTIF(event_name = 'click') AS pocket_clicks, COUNTIF(event_name = 'impression') AS pocket_impressions, @@ -249,9 +244,12 @@ events_unnested WHERE event_category = 'pocket' + AND event_name IN ('impression', 'click', 'save') GROUP BY newtab_visit_id, - pocket_story_position + pocket_story_position, + pocket_tile_id, + pocket_recommendation_id ), pocket_summary AS ( SELECT @@ -259,6 +257,8 @@ ARRAY_AGG( STRUCT( pocket_story_position, + pocket_tile_id, + pocket_recommendation_id, pocket_impressions, sponsored_pocket_impressions, organic_pocket_impressions, @@ -275,6 +275,99 @@ GROUP BY newtab_visit_id ), +wallpaper_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + mozfun.map.get_key(event_details, "selected_wallpaper") AS wallpaper_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_click') AS wallpaper_clicks, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "true" + ) AS wallpaper_clicks_had_previous_wallpaper, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "false" + ) AS wallpaper_clicks_first_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_category_click') AS wallpaper_category_clicks, + COUNTIF(event_name = 'wallpaper_highlight_dismissed') AS wallpaper_highlight_dismissals, + COUNTIF(event_name = 'wallpaper_highlight_cta_click') AS wallpaper_highlight_cta_clicks + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'wallpaper_click', + 'wallpaper_category_click', + 'wallpaper_highlight_cta_clicks', + 'wallpaper_highlight_dismissed' + ) + GROUP BY + newtab_visit_id, + wallpaper_selected_wallpaper +), +wallpaper_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + wallpaper_selected_wallpaper, + wallpaper_clicks, + wallpaper_clicks_had_previous_wallpaper, + wallpaper_clicks_first_selected_wallpaper, + wallpaper_category_clicks, + wallpaper_highlight_dismissals, + wallpaper_highlight_cta_clicks + ) + ) AS wallpaper_interactions + FROM + wallpaper_events + GROUP BY + newtab_visit_id +), +weather_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + COUNTIF(event_name = 'weather_impression') AS weather_widget_impressions, + COUNTIF(event_name = 'weather_open_provider_url') AS weather_widget_clicks, + COUNTIF(event_name = 'weather_load_error') AS weather_widget_load_errors, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "detailed" + ) AS weather_widget_change_display_to_detailed, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "simple" + ) AS weather_widget_change_display_to_simple + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'weather_impression', + 'weather_open_provider_url', + 'weather_load_error', + 'weather_change_display' + ) + GROUP BY + newtab_visit_id +), +weather_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + weather_widget_impressions, + weather_widget_clicks, + weather_widget_load_errors, + weather_widget_change_display_to_detailed, + weather_widget_change_display_to_simple + ) + ) AS weather_interactions + FROM + weather_events + GROUP BY + newtab_visit_id +), combined_newtab_activity AS ( SELECT * @@ -289,6 +382,12 @@ LEFT JOIN pocket_summary USING (newtab_visit_id) + LEFT JOIN + wallpaper_summary + USING (newtab_visit_id) + LEFT JOIN + weather_summary + USING (newtab_visit_id) WHERE -- Keep only rows with interactions, unless we receive a valid newtab.opened event. -- This is meant to drop only interactions that only have a newtab.closed event on the same partition @@ -297,6 +396,8 @@ OR search_interactions IS NOT NULL OR topsite_tile_interactions IS NOT NULL OR pocket_interactions IS NOT NULL + OR wallpaper_interactions IS NOT NULL + OR weather_interactions IS NOT NULL ), client_profile_info AS ( SELECT @@ -311,7 +412,15 @@ client_id ) SELECT - * + *, + CASE + WHEN ( + (newtab_open_source = "about:home" AND newtab_homepage_category = "enabled") + OR (newtab_open_source = "about:newtab" AND newtab_newtab_category = "enabled") + ) + THEN "default" + ELSE "non-default" + END AS newtab_default_ui, FROM combined_newtab_activity LEFT JOIN diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml 2024-06-13 20:06:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml 2024-06-13 20:07:25.000000000 +0000 @@ -215,6 +215,12 @@ - mode: NULLABLE name: organic_pocket_saves type: INTEGER + - name: pocket_tile_id + type: STRING + mode: NULLABLE + - name: pocket_recommendation_id + type: STRING + mode: NULLABLE mode: REPEATED name: pocket_interactions type: RECORD @@ -224,3 +230,53 @@ - mode: NULLABLE name: activity_segment type: STRING +- mode: NULLABLE + name: newtab_default_ui + type: STRING +- name: newtab_weather_widget_enabled + type: BOOLEAN + mode: NULLABLE +- name: wallpaper_interactions + type: RECORD + mode: REPEATED + fields: + - name: wallpaper_selected_wallpaper + type: STRING + mode: NULLABLE + - name: wallpaper_clicks + type: INTEGER + mode: NULLABLE + - name: wallpaper_clicks_had_previous_wallpaper + type: INTEGER + mode: NULLABLE + - name: wallpaper_clicks_first_selected_wallpaper + type: INTEGER + mode: NULLABLE + - name: wallpaper_category_clicks + type: INTEGER + mode: NULLABLE + - name: wallpaper_highlight_dismissals + type: INTEGER + mode: NULLABLE + - name: wallpaper_highlight_cta_clicks + type: INTEGER + mode: NULLABLE +- name: weather_interactions + type: RECORD + mode: REPEATED + fields: + - name: weather_widget_impressions + type: INTEGER + mode: NULLABLE + - name: weather_widget_clicks + type: INTEGER + mode: NULLABLE + - name: weather_widget_load_errors + type: INTEGER + mode: NULLABLE + - name: weather_widget_change_display_to_detailed + type: INTEGER + mode: NULLABLE + - name: weather_widget_change_display_to_simple + type: INTEGER + mode: NULLABLE ```

Link to full diff

dataops-ci-bot commented 2 weeks ago

Integration report for "Fix tests"

sql.diff

Click to expand! ```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare.py /tmp/workspace/generated-sql/dags/bqetl_cloudflare.py --- /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare.py 2024-06-13 21:10:04.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_cloudflare.py 2024-06-13 21:28:02.000000000 +0000 @@ -84,20 +84,3 @@ owner="kwindau@mozilla.com", email=["kwindau@mozilla.com"], ) - - cloudflare_derived__os_usage__v1 = GKEPodOperator( - task_id="cloudflare_derived__os_usage__v1", - arguments=[ - "python", - "sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py", - ] - + [ - "--date", - "{{ds}}", - "--cloudflare_api_token", - "{{ var.value.cloudflare_auth_token}}", - ], - image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest", - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com"], - ) Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1: query.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:06:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:09:12.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.events_v1` + `moz-fx-data-shared-prod.bedrock_live.interaction_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml 2024-06-13 21:07:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml 2024-06-13 21:24:46.000000000 +0000 @@ -6,15 +6,6 @@ labels: incremental: true owner1: kwindau - dag: bqetl_cloudflare -scheduling: - dag_name: bqetl_cloudflare - date_partition_parameter: dte - arguments: - - --date - - '{{ds}}' - - --cloudflare_api_token - - '{{ var.value.cloudflare_auth_token}}' bigquery: time_partitioning: type: day diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py 2024-06-13 21:06:07.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,385 +0,0 @@ -# Load libraries -import json -from datetime import datetime, timedelta -import pandas as pd -import requests -from argparse import ArgumentParser -from google.cloud import bigquery -from google.cloud import storage - -# Configurations -os_usg_configs = { - "timeout_limit": 2000, - "device_types": ["DESKTOP", "MOBILE", "OTHER", "ALL"], - "locations": [ - "ALL", - "BE", - "BG", - "CA", - "CZ", - "DE", - "DK", - "EE", - "ES", - "FI", - "FR", - "GB", - "HR", - "IE", - "IT", - "CY", - "LV", - "LT", - "LU", - "HU", - "MT", - "MX", - "NL", - "AT", - "PL", - "PT", - "RO", - "SI", - "SK", - "US", - "SE", - "GR", - ], - "bucket": "gs://moz-fx-data-prod-external-data/", - "bucket_no_gs": "moz-fx-data-prod-external-data", - "results_stg_gcs_fpth": "cloudflare/os_usage/RESULTS_STAGING/%s_results_from_%s_run.csv", - "results_archive_gcs_fpth": "cloudflare/os_usage/RESULTS_ARCHIVE/%s_results_from_%s_run.csv", - "errors_stg_gcs_fpth": "cloudflare/os_usage/ERRORS_STAGING/%s_results_from_%s_run.csv", - "errors_archive_gcs_fpth": "cloudflare/os_usage/ERRORS_ARCHIVE/%s_results_from_%s_run.csv", - "gcp_project_id": "moz-fx-data-shared-prod", - "gcp_conn_id": "google_cloud_shared_prod", - "results_bq_stg_table": "moz-fx-data-shared-prod.cloudflare_derived.os_results_stg", - "errors_bq_stg_table": "moz-fx-data-shared-prod.cloudflare_derived.os_errors_stg", -} - - -# Define a function to move a GCS object then delete the original -def move_blob(bucket_name, blob_name, destination_bucket_name, destination_blob_name): - """Moves a blob from one bucket to another with a new name.""" - storage_client = storage.Client() - source_bucket = storage_client.bucket(bucket_name) - source_blob = source_bucket.blob(blob_name) - destination_bucket = storage_client.bucket(destination_bucket_name) - destination_generation_match_precondition = 0 - - blob_copy = source_bucket.copy_blob( - source_blob, - destination_bucket, - destination_blob_name, - if_generation_match=destination_generation_match_precondition, - ) - source_bucket.delete_blob(blob_name) - - print( - "Blob {} in bucket {} moved to blob {} in bucket {}.".format( - source_blob.name, - source_bucket.name, - blob_copy.name, - destination_bucket.name, - ) - ) - - -# Function to configure the API URL -def generate_os_timeseries_api_call(strt_dt, end_dt, agg_int, location, device_type): - """Generate the API call for Operating System Usage Data.""" - if location == "ALL" and device_type == "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&format=json&aggInterval={agg_int}" - elif location != "ALL" and device_type == "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={strt_dt}T00:00:00.000Z&location={location}&format=json&aggInterval={agg_int}" - elif location == "ALL" and device_type != "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&deviceType={device_type}&format=json&aggInterval={agg_int}" - else: - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&location={location}&deviceType={device_type}&format=json&aggInterval={agg_int}" - return os_usage_api_url - - -def get_os_usage_data(date_of_interest, auth_token): - """Pull OS usage data from the Cloudflare API and save errors & results to GCS.""" - # Calculate start date and end date - logical_dag_dt = date_of_interest - logical_dag_dt_as_date = datetime.strptime(logical_dag_dt, "%Y-%m-%d").date() - start_date = logical_dag_dt_as_date - timedelta(days=4) - end_date = start_date + timedelta(days=1) - print("Start Date: ", start_date) - print("End Date: ", end_date) - - # Configure request headers - bearer_string = f"Bearer {auth_token}" - headers = {"Authorization": bearer_string} - - # Initialize the empty results & errors dataframe - result_df = pd.DataFrame( - { - "Timestamps": [], - "OS": [], - "Location": [], - "DeviceType": [], - "Share": [], - "ConfidenceLevel": [], - "AggrInterval": [], - "Normalization": [], - "LastUpdatedTS": [], - } - ) - - # Initialize an errors dataframe - errors_df = pd.DataFrame( - {"StartTime": [], "EndTime": [], "Location": [], "DeviceType": []} - ) - - # Go through all combinations, submit API requests - for device_type in os_usg_configs["device_types"]: - for loc in os_usg_configs["locations"]: - print("Device Type: ", device_type) - print("Loc: ", loc) - - # Generate the URL with given parameters - os_usage_api_url = generate_os_timeseries_api_call( - start_date, end_date, "1d", loc, device_type - ) - try: - # Call the API and save the response as JSON - response = requests.get( - os_usage_api_url, - headers=headers, - timeout=os_usg_configs["timeout_limit"], - ) - response_json = json.loads(response.text) - - # If response was successful, get the result - if response_json["success"] is True: - result = response_json["result"] - # Parse metadata - conf_lvl = result["meta"]["confidenceInfo"]["level"] - aggr_intvl = result["meta"]["aggInterval"] - nrmlztn = result["meta"]["normalization"] - lst_upd = result["meta"]["lastUpdated"] - data_dict = result["serie_0"] - - for key, val in data_dict.items(): - if key != 'timestamps': - new_result_df = pd.DataFrame( - { - "Timestamps": data_dict["timestamps"], - "OS": [key] * len(val), - "Location": [loc] * len(val), - "DeviceType": [device_type] * len(val), - "Share": val, - "ConfidenceLevel": [conf_lvl] * len(val), - "AggrInterval": [aggr_intvl] * len(val), - "Normalization": [nrmlztn] * len(val), - "LastUpdatedTS": [lst_upd] * len(val), - } - ) - result_df = pd.concat([result_df, new_result_df]) - - # If response was not successful, get the errors - else: - # errors = response_json["errors"] # Maybe add to capture, right now not using this - new_errors_df = pd.DataFrame( - { - "StartTime": [start_date], - "EndTime": [end_date], - "Location": [loc], - "DeviceType": [device_type], - } - ) - errors_df = pd.concat([errors_df, new_errors_df]) - except: - new_errors_df = pd.DataFrame( - { - "StartTime": [start_date], - "EndTime": [end_date], - "Location": [loc], - "DeviceType": [device_type], - } - ) - errors_df = pd.concat([errors_df, new_errors_df]) - - result_fpath = os_usg_configs["bucket"] + os_usg_configs["results_stg_gcs_fpth"] % ( - start_date, - logical_dag_dt, - ) - errors_fpath = os_usg_configs["bucket"] + os_usg_configs["errors_stg_gcs_fpth"] % ( - start_date, - logical_dag_dt, - ) - - result_df.to_csv(result_fpath, index=False) - errors_df.to_csv(errors_fpath, index=False) - print("Wrote errors to: ", errors_fpath) - print("Wrote results to: ", result_fpath) - - # Write a summary to the logs - len_results = str(len(result_df)) - len_errors = str(len(errors_df)) - result_summary = f"# Result Rows: {len_results}; # of Error Rows: {len_errors}" - return result_summary - - -def main(): - """Call the API, save data to GCS, load to BQ staging, delete & load to BQ gold""" - parser = ArgumentParser(description=__doc__) - parser.add_argument("--date", required=True) - parser.add_argument("--cloudflare_api_token", required=True) - parser.add_argument("--project", default="moz-fx-data-shared-prod") - parser.add_argument("--dataset", default="cloudflare_derived") - - args = parser.parse_args() - print("Running for date: ") - print(args.date) - - # STEP 1 - Pull the data from the API, save results & errors to GCS staging area - result_summary = get_os_usage_data(args.date, args.cloudflare_api_token) - print("result_summary") - print(result_summary) - - # Create a bigquery client - client = bigquery.Client(args.project) - - result_uri = os_usg_configs["bucket"] + os_usg_configs["results_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - error_uri = os_usg_configs["bucket"] + os_usg_configs["errors_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - print("result_uri") - print(result_uri) - - print("error_uri") - print(error_uri) - - # STEP 2 - Copy the result data from GCS staging to BQ staging table - load_result_csv_to_bq_stg_job = client.load_table_from_uri( - result_uri, - os_usg_configs["results_bq_stg_table"], - job_config=bigquery.LoadJobConfig( - create_disposition="CREATE_IF_NEEDED", - write_disposition="WRITE_TRUNCATE", - schema=[ - {"name": "Timestamps", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "OS", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Location", "type": "STRING", "mode": "NULLABLE"}, - {"name": "DeviceType", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Share", "type": "NUMERIC", "mode": "NULLABLE"}, - {"name": "ConfidenceLevel", "type": "STRING", "mode": "NULLABLE"}, - {"name": "AggrInterval", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Normalization", "type": "STRING", "mode": "NULLABLE"}, - {"name": "LastUpdatedTS", "type": "TIMESTAMP", "mode": "NULLABLE"}, - ], - skip_leading_rows=1, - source_format=bigquery.SourceFormat.CSV, - ), - ) - - load_result_csv_to_bq_stg_job.result() - result_bq_stg_tbl = client.get_table(os_usg_configs["results_bq_stg_table"]) - print("Loaded {} rows to results staging.".format(result_bq_stg_tbl.num_rows)) - - # STEP 3 - Copy the error data from GCS staging to BQ staging table - load_error_csv_to_bq_stg_job = client.load_table_from_uri( - error_uri, - os_usg_configs["errors_bq_stg_table"], - job_config=bigquery.LoadJobConfig( - create_disposition="CREATE_IF_NEEDED", - write_disposition="WRITE_TRUNCATE", - schema=[ - {"name": "StartTime", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "EndTime", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "Location", "type": "STRING", "mode": "NULLABLE"}, - {"name": "DeviceType", "type": "STRING", "mode": "NULLABLE"}, - ], - skip_leading_rows=1, - source_format=bigquery.SourceFormat.CSV, - ), - ) - - load_error_csv_to_bq_stg_job.result() - error_bq_stg_tbl = client.get_table(os_usg_configs["errors_bq_stg_table"]) - print("Loaded {} rows to errors staging.".format(error_bq_stg_tbl.num_rows)) - - # STEP 4 - Delete results from gold for this day, if there are any already (so if rerun, no issues will occur) - del_exstng_gold_res_for_date = f"""DELETE FROM `moz-fx-data-shared-prod.cloudflare_derived.os_usage_v1` WHERE dte = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - del_gold_res_job = client.query(del_exstng_gold_res_for_date) - del_gold_res_job.result() - print("Deleted anything already existing for this date from results gold") - - # STEP 5 - Delete errors from gold for this day, if there are any already (so if rerun, no issues will occur) - del_exstng_gold_err_for_date = f"""DELETE FROM `moz-fx-data-shared-prod.cloudflare_derived.os_usage_errors_v1` WHERE dte = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - del_gold_err_job = client.query(del_exstng_gold_err_for_date) - del_gold_err_job.result() - print("Deleted anything already existing for this date from errors gold") - - # STEP 6 - Load results from stage to gold # NEED TO UPDATE THIS STILL - device_usg_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_v1` -SELECT -CAST(Timestamps AS date) AS dte, -OS AS os, -Location AS location, -DeviceType AS device_type, -Share AS os_share, -Normalization AS normalization_type, -LastUpdatedTS AS last_updated_ts -FROM `moz-fx-data-shared-prod.cloudflare_derived.os_results_stg` -WHERE CAST(Timestamps as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - load_res_to_gold = client.query(device_usg_stg_to_gold_query) - load_res_to_gold.result() - - # STEP 7 - Load errors from stage to gold - browser_usg_errors_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_errors_v1` -SELECT -CAST(StartTime as date) AS dte, -Location AS location, -DeviceType AS device_type -FROM `moz-fx-data-shared-prod.cloudflare_derived.os_errors_stg` -WHERE CAST(StartTime as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - load_err_to_gold = client.query(browser_usg_errors_stg_to_gold_query) - load_err_to_gold.result() - - # Initialize a storage client to use in next steps - storage_client = storage.Client() - - # STEP 8 - Copy the result CSV from stage to archive, then delete from stage - # Calculate the fpaths we will use ahead of time - result_stg_fpath = os_usg_configs["results_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - result_archive_fpath = os_usg_configs["results_archive_gcs_fpath"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - move_blob( - os_usg_configs["bucket_no_gs"], - result_stg_fpath, - os_usg_configs["bucket_no_gs"], - result_archive_fpath, - ) - - # STEP 9 - Copy the error CSV from stage to archive, then delete from stage - error_stg_fpath = os_usg_configs["errors_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - error_archive_fpath = os_usg_configs["errors_archive_gcs_fpath"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - move_blob( - os_usg_configs["bucket_no_gs"], - error_stg_fpath, - os_usg_configs["bucket_no_gs"], - error_archive_fpath, - ) - - -if __name__ == "__main__": - main() diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-13 21:06:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-13 21:18:08.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: source + mode: NULLABLE +- name: source type: STRING -- mode: NULLABLE - name: event_type + mode: NULLABLE +- name: event_type type: STRING -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: subdivision1 + mode: NULLABLE +- name: subdivision1 type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: event_count + mode: NULLABLE +- name: event_count type: INTEGER -- mode: NULLABLE - name: user_count + mode: NULLABLE +- name: user_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-13 21:06:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-13 21:18:08.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: impression_count + mode: NULLABLE +- name: impression_count type: INTEGER -- mode: NULLABLE - name: click_count + mode: NULLABLE +- name: click_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-13 21:06:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-13 21:15:48.000000000 +0000 @@ -26,6 +26,9 @@ - name: adjust_network type: STRING mode: NULLABLE +- name: install_source + type: STRING + mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-13 21:06:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-13 21:15:50.000000000 +0000 @@ -48,6 +48,10 @@ description: 'The type of source of a client installation. ' +- name: install_source + type: STRING + mode: NULLABLE + description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:06:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:09:13.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:06:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:09:13.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -80,7 +80,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-13 21:06:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-13 21:10:52.000000000 +0000 @@ -45,7 +45,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -75,7 +75,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -582,7 +582,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -592,7 +592,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -673,7 +673,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -683,7 +683,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -764,7 +764,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -774,7 +774,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1555,7 +1555,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1565,7 +1565,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1646,7 +1646,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1656,7 +1656,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1737,7 +1737,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1747,7 +1747,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1828,7 +1828,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1838,7 +1838,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1990,7 +1990,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2000,7 +2000,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -2010,7 +2010,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.events_v1` + `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2162,7 +2162,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -2172,7 +2172,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:06:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:09:12.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-13 21:06:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-13 21:14:48.000000000 +0000 @@ -1,7 +1,13 @@ fields: -- type: DATETIME - name: build_hour -- type: INTEGER - name: geckoview_major_version -- type: INTEGER - name: n_pings +- name: build_hour + type: DATETIME + mode: NULLABLE + description: null +- name: geckoview_major_version + type: INTEGER + mode: NULLABLE + description: null +- name: n_pings + type: INTEGER + mode: NULLABLE + description: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta/metrics/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta/metrics/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta/metrics/schema.yaml 2024-06-13 21:06:45.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta/metrics/schema.yaml 2024-06-13 21:08:34.000000000 +0000 @@ -3265,7 +3265,8 @@ - name: value type: INTEGER mode: NULLABLE - description: 'Failure occurs when initializing the audio stream. + description: 'Failure occurs when initializing the audio stream. (Migrated from + the geckoview metric of the same name). ' - name: glean_validation_pings_submitted diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:06:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:09:14.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:06:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:09:15.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:06:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:09:15.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:06:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:09:15.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:06:45.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:09:15.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:06:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:09:15.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/windows_10_aggregate/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/windows_10_aggregate/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/windows_10_aggregate/schema.yaml 2024-06-13 21:06:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/windows_10_aggregate/schema.yaml 2024-06-13 21:15:01.000000000 +0000 @@ -20,7 +20,6 @@ - name: normalized_channel type: STRING mode: NULLABLE - description: null - name: count type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/windows_10_build_distribution/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/windows_10_build_distribution/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/windows_10_build_distribution/schema.yaml 2024-06-13 21:06:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/windows_10_build_distribution/schema.yaml 2024-06-13 21:15:03.000000000 +0000 @@ -2,7 +2,6 @@ - name: normalized_channel type: STRING mode: NULLABLE - description: null - name: build_group type: STRING mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql 2024-06-13 21:06:07.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql 2024-06-13 21:07:17.000000000 +0000 @@ -20,16 +20,6 @@ WHERE DATE(submission_timestamp) = @submission_date AND category IN ('newtab', 'topsites', 'newtab.search', 'newtab.search.ad', 'pocket') - AND name IN ( - 'closed', - 'opened', - 'impression', - 'issued', - 'click', - 'save', - 'topic_click', - 'dismiss' - ) ), visit_metadata AS ( SELECT @@ -54,6 +44,7 @@ ANY_VALUE(metrics.string.newtab_homepage_category) AS newtab_homepage_category, ANY_VALUE(metrics.string.newtab_newtab_category) AS newtab_newtab_category, ANY_VALUE(metrics.boolean.newtab_search_enabled) AS newtab_search_enabled, + ANY_VALUE(metrics.boolean.newtab_weather_enabled) AS newtab_weather_widget_enabled, ANY_VALUE(metrics.quantity.topsites_rows) AS topsites_rows, ANY_VALUE(metrics.string_list.newtab_blocked_sponsors) AS newtab_blocked_sponsors, ANY_VALUE(ping_info.experiments) AS experiments, @@ -110,6 +101,7 @@ events_unnested WHERE event_category IN ('newtab.search', 'newtab.search.ad') + AND event_name IN ('click', 'impression', 'issued') GROUP BY newtab_visit_id, search_engine, @@ -180,6 +172,7 @@ ) WHERE event_category = 'topsites' + AND event_name IN ('dismiss', 'click', 'impression') GROUP BY newtab_visit_id, topsite_tile_position, @@ -218,6 +211,8 @@ SELECT mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS pocket_story_position, + mozfun.map.get_key(event_details, "tile_id") AS pocket_tile_id, + mozfun.map.get_key(event_details, "recommendation_id") AS pocket_recommendation_id, COUNTIF(event_name = 'save') AS pocket_saves, COUNTIF(event_name = 'click') AS pocket_clicks, COUNTIF(event_name = 'impression') AS pocket_impressions, @@ -249,9 +244,12 @@ events_unnested WHERE event_category = 'pocket' + AND event_name IN ('impression', 'click', 'save') GROUP BY newtab_visit_id, - pocket_story_position + pocket_story_position, + pocket_tile_id, + pocket_recommendation_id ), pocket_summary AS ( SELECT @@ -259,6 +257,8 @@ ARRAY_AGG( STRUCT( pocket_story_position, + pocket_tile_id, + pocket_recommendation_id, pocket_impressions, sponsored_pocket_impressions, organic_pocket_impressions, @@ -275,6 +275,99 @@ GROUP BY newtab_visit_id ), +wallpaper_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + mozfun.map.get_key(event_details, "selected_wallpaper") AS wallpaper_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_click') AS wallpaper_clicks, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "true" + ) AS wallpaper_clicks_had_previous_wallpaper, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "false" + ) AS wallpaper_clicks_first_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_category_click') AS wallpaper_category_clicks, + COUNTIF(event_name = 'wallpaper_highlight_dismissed') AS wallpaper_highlight_dismissals, + COUNTIF(event_name = 'wallpaper_highlight_cta_click') AS wallpaper_highlight_cta_clicks + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'wallpaper_click', + 'wallpaper_category_click', + 'wallpaper_highlight_cta_clicks', + 'wallpaper_highlight_dismissed' + ) + GROUP BY + newtab_visit_id, + wallpaper_selected_wallpaper +), +wallpaper_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + wallpaper_selected_wallpaper, + wallpaper_clicks, + wallpaper_clicks_had_previous_wallpaper, + wallpaper_clicks_first_selected_wallpaper, + wallpaper_category_clicks, + wallpaper_highlight_dismissals, + wallpaper_highlight_cta_clicks + ) + ) AS wallpaper_interactions + FROM + wallpaper_events + GROUP BY + newtab_visit_id +), +weather_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + COUNTIF(event_name = 'weather_impression') AS weather_widget_impressions, + COUNTIF(event_name = 'weather_open_provider_url') AS weather_widget_clicks, + COUNTIF(event_name = 'weather_load_error') AS weather_widget_load_errors, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "detailed" + ) AS weather_widget_change_display_to_detailed, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "simple" + ) AS weather_widget_change_display_to_simple + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'weather_impression', + 'weather_open_provider_url', + 'weather_load_error', + 'weather_change_display' + ) + GROUP BY + newtab_visit_id +), +weather_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + weather_widget_impressions, + weather_widget_clicks, + weather_widget_load_errors, + weather_widget_change_display_to_detailed, + weather_widget_change_display_to_simple + ) + ) AS weather_interactions + FROM + weather_events + GROUP BY + newtab_visit_id +), combined_newtab_activity AS ( SELECT * @@ -289,6 +382,12 @@ LEFT JOIN pocket_summary USING (newtab_visit_id) + LEFT JOIN + wallpaper_summary + USING (newtab_visit_id) + LEFT JOIN + weather_summary + USING (newtab_visit_id) WHERE -- Keep only rows with interactions, unless we receive a valid newtab.opened event. -- This is meant to drop only interactions that only have a newtab.closed event on the same partition @@ -297,6 +396,8 @@ OR search_interactions IS NOT NULL OR topsite_tile_interactions IS NOT NULL OR pocket_interactions IS NOT NULL + OR wallpaper_interactions IS NOT NULL + OR weather_interactions IS NOT NULL ), client_profile_info AS ( SELECT @@ -311,7 +412,15 @@ client_id ) SELECT - * + *, + CASE + WHEN ( + (newtab_open_source = "about:home" AND newtab_homepage_category = "enabled") + OR (newtab_open_source = "about:newtab" AND newtab_newtab_category = "enabled") + ) + THEN "default" + ELSE "non-default" + END AS newtab_default_ui, FROM combined_newtab_activity LEFT JOIN diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml 2024-06-13 21:06:07.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml 2024-06-13 21:07:17.000000000 +0000 @@ -215,6 +215,12 @@ - mode: NULLABLE name: organic_pocket_saves type: INTEGER + - name: pocket_tile_id + type: STRING + mode: NULLABLE + - name: pocket_recommendation_id + type: STRING + mode: NULLABLE mode: REPEATED name: pocket_interactions type: RECORD @@ -224,3 +230,53 @@ - mode: NULLABLE name: activity_segment type: STRING +- mode: NULLABLE + name: newtab_default_ui + type: STRING +- name: newtab_weather_widget_enabled + type: BOOLEAN + mode: NULLABLE +- name: wallpaper_interactions + type: RECORD + mode: REPEATED + fields: + - name: wallpaper_selected_wallpaper + type: STRING + mode: NULLABLE + - name: wallpaper_clicks + type: INTEGER + mode: NULLABLE + - name: wallpaper_clicks_had_previous_wallpaper + type: INTEGER + mode: NULLABLE + - name: wallpaper_clicks_first_selected_wallpaper + type: INTEGER + mode: NULLABLE + - name: wallpaper_category_clicks + type: INTEGER + mode: NULLABLE + - name: wallpaper_highlight_dismissals + type: INTEGER + mode: NULLABLE + - name: wallpaper_highlight_cta_clicks + type: INTEGER + mode: NULLABLE +- name: weather_interactions + type: RECORD + mode: REPEATED + fields: + - name: weather_widget_impressions + type: INTEGER + mode: NULLABLE + - name: weather_widget_clicks + type: INTEGER + mode: NULLABLE + - name: weather_widget_load_errors + type: INTEGER + mode: NULLABLE + - name: weather_widget_change_display_to_detailed + type: INTEGER + mode: NULLABLE + - name: weather_widget_change_display_to_simple + type: INTEGER + mode: NULLABLE ```

Link to full diff

dataops-ci-bot commented 2 weeks ago

Integration report for "Add category metric to test values"

sql.diff

Click to expand! ```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare.py /tmp/workspace/generated-sql/dags/bqetl_cloudflare.py --- /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare.py 2024-06-13 21:51:33.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_cloudflare.py 2024-06-13 22:09:29.000000000 +0000 @@ -84,20 +84,3 @@ owner="kwindau@mozilla.com", email=["kwindau@mozilla.com"], ) - - cloudflare_derived__os_usage__v1 = GKEPodOperator( - task_id="cloudflare_derived__os_usage__v1", - arguments=[ - "python", - "sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py", - ] - + [ - "--date", - "{{ds}}", - "--cloudflare_api_token", - "{{ var.value.cloudflare_auth_token}}", - ], - image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest", - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com"], - ) Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1: query.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:48:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:50:58.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.interaction_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml 2024-06-13 21:49:42.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml 2024-06-13 22:06:29.000000000 +0000 @@ -6,15 +6,6 @@ labels: incremental: true owner1: kwindau - dag: bqetl_cloudflare -scheduling: - dag_name: bqetl_cloudflare - date_partition_parameter: dte - arguments: - - --date - - '{{ds}}' - - --cloudflare_api_token - - '{{ var.value.cloudflare_auth_token}}' bigquery: time_partitioning: type: day diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py 2024-06-13 21:48:09.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,385 +0,0 @@ -# Load libraries -import json -from datetime import datetime, timedelta -import pandas as pd -import requests -from argparse import ArgumentParser -from google.cloud import bigquery -from google.cloud import storage - -# Configurations -os_usg_configs = { - "timeout_limit": 2000, - "device_types": ["DESKTOP", "MOBILE", "OTHER", "ALL"], - "locations": [ - "ALL", - "BE", - "BG", - "CA", - "CZ", - "DE", - "DK", - "EE", - "ES", - "FI", - "FR", - "GB", - "HR", - "IE", - "IT", - "CY", - "LV", - "LT", - "LU", - "HU", - "MT", - "MX", - "NL", - "AT", - "PL", - "PT", - "RO", - "SI", - "SK", - "US", - "SE", - "GR", - ], - "bucket": "gs://moz-fx-data-prod-external-data/", - "bucket_no_gs": "moz-fx-data-prod-external-data", - "results_stg_gcs_fpth": "cloudflare/os_usage/RESULTS_STAGING/%s_results_from_%s_run.csv", - "results_archive_gcs_fpth": "cloudflare/os_usage/RESULTS_ARCHIVE/%s_results_from_%s_run.csv", - "errors_stg_gcs_fpth": "cloudflare/os_usage/ERRORS_STAGING/%s_results_from_%s_run.csv", - "errors_archive_gcs_fpth": "cloudflare/os_usage/ERRORS_ARCHIVE/%s_results_from_%s_run.csv", - "gcp_project_id": "moz-fx-data-shared-prod", - "gcp_conn_id": "google_cloud_shared_prod", - "results_bq_stg_table": "moz-fx-data-shared-prod.cloudflare_derived.os_results_stg", - "errors_bq_stg_table": "moz-fx-data-shared-prod.cloudflare_derived.os_errors_stg", -} - - -# Define a function to move a GCS object then delete the original -def move_blob(bucket_name, blob_name, destination_bucket_name, destination_blob_name): - """Moves a blob from one bucket to another with a new name.""" - storage_client = storage.Client() - source_bucket = storage_client.bucket(bucket_name) - source_blob = source_bucket.blob(blob_name) - destination_bucket = storage_client.bucket(destination_bucket_name) - destination_generation_match_precondition = 0 - - blob_copy = source_bucket.copy_blob( - source_blob, - destination_bucket, - destination_blob_name, - if_generation_match=destination_generation_match_precondition, - ) - source_bucket.delete_blob(blob_name) - - print( - "Blob {} in bucket {} moved to blob {} in bucket {}.".format( - source_blob.name, - source_bucket.name, - blob_copy.name, - destination_bucket.name, - ) - ) - - -# Function to configure the API URL -def generate_os_timeseries_api_call(strt_dt, end_dt, agg_int, location, device_type): - """Generate the API call for Operating System Usage Data.""" - if location == "ALL" and device_type == "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&format=json&aggInterval={agg_int}" - elif location != "ALL" and device_type == "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={strt_dt}T00:00:00.000Z&location={location}&format=json&aggInterval={agg_int}" - elif location == "ALL" and device_type != "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&deviceType={device_type}&format=json&aggInterval={agg_int}" - else: - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&location={location}&deviceType={device_type}&format=json&aggInterval={agg_int}" - return os_usage_api_url - - -def get_os_usage_data(date_of_interest, auth_token): - """Pull OS usage data from the Cloudflare API and save errors & results to GCS.""" - # Calculate start date and end date - logical_dag_dt = date_of_interest - logical_dag_dt_as_date = datetime.strptime(logical_dag_dt, "%Y-%m-%d").date() - start_date = logical_dag_dt_as_date - timedelta(days=4) - end_date = start_date + timedelta(days=1) - print("Start Date: ", start_date) - print("End Date: ", end_date) - - # Configure request headers - bearer_string = f"Bearer {auth_token}" - headers = {"Authorization": bearer_string} - - # Initialize the empty results & errors dataframe - result_df = pd.DataFrame( - { - "Timestamps": [], - "OS": [], - "Location": [], - "DeviceType": [], - "Share": [], - "ConfidenceLevel": [], - "AggrInterval": [], - "Normalization": [], - "LastUpdatedTS": [], - } - ) - - # Initialize an errors dataframe - errors_df = pd.DataFrame( - {"StartTime": [], "EndTime": [], "Location": [], "DeviceType": []} - ) - - # Go through all combinations, submit API requests - for device_type in os_usg_configs["device_types"]: - for loc in os_usg_configs["locations"]: - print("Device Type: ", device_type) - print("Loc: ", loc) - - # Generate the URL with given parameters - os_usage_api_url = generate_os_timeseries_api_call( - start_date, end_date, "1d", loc, device_type - ) - try: - # Call the API and save the response as JSON - response = requests.get( - os_usage_api_url, - headers=headers, - timeout=os_usg_configs["timeout_limit"], - ) - response_json = json.loads(response.text) - - # If response was successful, get the result - if response_json["success"] is True: - result = response_json["result"] - # Parse metadata - conf_lvl = result["meta"]["confidenceInfo"]["level"] - aggr_intvl = result["meta"]["aggInterval"] - nrmlztn = result["meta"]["normalization"] - lst_upd = result["meta"]["lastUpdated"] - data_dict = result["serie_0"] - - for key, val in data_dict.items(): - if key != 'timestamps': - new_result_df = pd.DataFrame( - { - "Timestamps": data_dict["timestamps"], - "OS": [key] * len(val), - "Location": [loc] * len(val), - "DeviceType": [device_type] * len(val), - "Share": val, - "ConfidenceLevel": [conf_lvl] * len(val), - "AggrInterval": [aggr_intvl] * len(val), - "Normalization": [nrmlztn] * len(val), - "LastUpdatedTS": [lst_upd] * len(val), - } - ) - result_df = pd.concat([result_df, new_result_df]) - - # If response was not successful, get the errors - else: - # errors = response_json["errors"] # Maybe add to capture, right now not using this - new_errors_df = pd.DataFrame( - { - "StartTime": [start_date], - "EndTime": [end_date], - "Location": [loc], - "DeviceType": [device_type], - } - ) - errors_df = pd.concat([errors_df, new_errors_df]) - except: - new_errors_df = pd.DataFrame( - { - "StartTime": [start_date], - "EndTime": [end_date], - "Location": [loc], - "DeviceType": [device_type], - } - ) - errors_df = pd.concat([errors_df, new_errors_df]) - - result_fpath = os_usg_configs["bucket"] + os_usg_configs["results_stg_gcs_fpth"] % ( - start_date, - logical_dag_dt, - ) - errors_fpath = os_usg_configs["bucket"] + os_usg_configs["errors_stg_gcs_fpth"] % ( - start_date, - logical_dag_dt, - ) - - result_df.to_csv(result_fpath, index=False) - errors_df.to_csv(errors_fpath, index=False) - print("Wrote errors to: ", errors_fpath) - print("Wrote results to: ", result_fpath) - - # Write a summary to the logs - len_results = str(len(result_df)) - len_errors = str(len(errors_df)) - result_summary = f"# Result Rows: {len_results}; # of Error Rows: {len_errors}" - return result_summary - - -def main(): - """Call the API, save data to GCS, load to BQ staging, delete & load to BQ gold""" - parser = ArgumentParser(description=__doc__) - parser.add_argument("--date", required=True) - parser.add_argument("--cloudflare_api_token", required=True) - parser.add_argument("--project", default="moz-fx-data-shared-prod") - parser.add_argument("--dataset", default="cloudflare_derived") - - args = parser.parse_args() - print("Running for date: ") - print(args.date) - - # STEP 1 - Pull the data from the API, save results & errors to GCS staging area - result_summary = get_os_usage_data(args.date, args.cloudflare_api_token) - print("result_summary") - print(result_summary) - - # Create a bigquery client - client = bigquery.Client(args.project) - - result_uri = os_usg_configs["bucket"] + os_usg_configs["results_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - error_uri = os_usg_configs["bucket"] + os_usg_configs["errors_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - print("result_uri") - print(result_uri) - - print("error_uri") - print(error_uri) - - # STEP 2 - Copy the result data from GCS staging to BQ staging table - load_result_csv_to_bq_stg_job = client.load_table_from_uri( - result_uri, - os_usg_configs["results_bq_stg_table"], - job_config=bigquery.LoadJobConfig( - create_disposition="CREATE_IF_NEEDED", - write_disposition="WRITE_TRUNCATE", - schema=[ - {"name": "Timestamps", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "OS", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Location", "type": "STRING", "mode": "NULLABLE"}, - {"name": "DeviceType", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Share", "type": "NUMERIC", "mode": "NULLABLE"}, - {"name": "ConfidenceLevel", "type": "STRING", "mode": "NULLABLE"}, - {"name": "AggrInterval", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Normalization", "type": "STRING", "mode": "NULLABLE"}, - {"name": "LastUpdatedTS", "type": "TIMESTAMP", "mode": "NULLABLE"}, - ], - skip_leading_rows=1, - source_format=bigquery.SourceFormat.CSV, - ), - ) - - load_result_csv_to_bq_stg_job.result() - result_bq_stg_tbl = client.get_table(os_usg_configs["results_bq_stg_table"]) - print("Loaded {} rows to results staging.".format(result_bq_stg_tbl.num_rows)) - - # STEP 3 - Copy the error data from GCS staging to BQ staging table - load_error_csv_to_bq_stg_job = client.load_table_from_uri( - error_uri, - os_usg_configs["errors_bq_stg_table"], - job_config=bigquery.LoadJobConfig( - create_disposition="CREATE_IF_NEEDED", - write_disposition="WRITE_TRUNCATE", - schema=[ - {"name": "StartTime", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "EndTime", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "Location", "type": "STRING", "mode": "NULLABLE"}, - {"name": "DeviceType", "type": "STRING", "mode": "NULLABLE"}, - ], - skip_leading_rows=1, - source_format=bigquery.SourceFormat.CSV, - ), - ) - - load_error_csv_to_bq_stg_job.result() - error_bq_stg_tbl = client.get_table(os_usg_configs["errors_bq_stg_table"]) - print("Loaded {} rows to errors staging.".format(error_bq_stg_tbl.num_rows)) - - # STEP 4 - Delete results from gold for this day, if there are any already (so if rerun, no issues will occur) - del_exstng_gold_res_for_date = f"""DELETE FROM `moz-fx-data-shared-prod.cloudflare_derived.os_usage_v1` WHERE dte = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - del_gold_res_job = client.query(del_exstng_gold_res_for_date) - del_gold_res_job.result() - print("Deleted anything already existing for this date from results gold") - - # STEP 5 - Delete errors from gold for this day, if there are any already (so if rerun, no issues will occur) - del_exstng_gold_err_for_date = f"""DELETE FROM `moz-fx-data-shared-prod.cloudflare_derived.os_usage_errors_v1` WHERE dte = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - del_gold_err_job = client.query(del_exstng_gold_err_for_date) - del_gold_err_job.result() - print("Deleted anything already existing for this date from errors gold") - - # STEP 6 - Load results from stage to gold # NEED TO UPDATE THIS STILL - device_usg_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_v1` -SELECT -CAST(Timestamps AS date) AS dte, -OS AS os, -Location AS location, -DeviceType AS device_type, -Share AS os_share, -Normalization AS normalization_type, -LastUpdatedTS AS last_updated_ts -FROM `moz-fx-data-shared-prod.cloudflare_derived.os_results_stg` -WHERE CAST(Timestamps as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - load_res_to_gold = client.query(device_usg_stg_to_gold_query) - load_res_to_gold.result() - - # STEP 7 - Load errors from stage to gold - browser_usg_errors_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_errors_v1` -SELECT -CAST(StartTime as date) AS dte, -Location AS location, -DeviceType AS device_type -FROM `moz-fx-data-shared-prod.cloudflare_derived.os_errors_stg` -WHERE CAST(StartTime as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - load_err_to_gold = client.query(browser_usg_errors_stg_to_gold_query) - load_err_to_gold.result() - - # Initialize a storage client to use in next steps - storage_client = storage.Client() - - # STEP 8 - Copy the result CSV from stage to archive, then delete from stage - # Calculate the fpaths we will use ahead of time - result_stg_fpath = os_usg_configs["results_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - result_archive_fpath = os_usg_configs["results_archive_gcs_fpath"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - move_blob( - os_usg_configs["bucket_no_gs"], - result_stg_fpath, - os_usg_configs["bucket_no_gs"], - result_archive_fpath, - ) - - # STEP 9 - Copy the error CSV from stage to archive, then delete from stage - error_stg_fpath = os_usg_configs["errors_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - error_archive_fpath = os_usg_configs["errors_archive_gcs_fpath"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - move_blob( - os_usg_configs["bucket_no_gs"], - error_stg_fpath, - os_usg_configs["bucket_no_gs"], - error_archive_fpath, - ) - - -if __name__ == "__main__": - main() diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-13 21:48:08.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-13 21:56:34.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: source + mode: NULLABLE +- name: source type: STRING -- mode: NULLABLE - name: event_type + mode: NULLABLE +- name: event_type type: STRING -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: subdivision1 + mode: NULLABLE +- name: subdivision1 type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: event_count + mode: NULLABLE +- name: event_count type: INTEGER -- mode: NULLABLE - name: user_count + mode: NULLABLE +- name: user_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-13 21:48:08.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-13 21:56:33.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: impression_count + mode: NULLABLE +- name: impression_count type: INTEGER -- mode: NULLABLE - name: click_count + mode: NULLABLE +- name: click_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-13 21:48:08.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-13 21:56:36.000000000 +0000 @@ -26,6 +26,9 @@ - name: adjust_network type: STRING mode: NULLABLE +- name: install_source + type: STRING + mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-13 21:48:08.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-13 21:56:37.000000000 +0000 @@ -48,6 +48,10 @@ description: 'The type of source of a client installation. ' +- name: install_source + type: STRING + mode: NULLABLE + description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:48:45.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:50:59.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:48:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:50:59.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -80,7 +80,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-13 21:48:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-13 21:52:44.000000000 +0000 @@ -45,7 +45,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -75,7 +75,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -572,7 +572,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -582,7 +582,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -592,7 +592,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -663,7 +663,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -673,7 +673,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -683,7 +683,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -754,7 +754,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -764,7 +764,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -774,7 +774,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1555,7 +1555,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1575,7 +1575,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1646,7 +1646,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1666,7 +1666,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1737,7 +1737,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1757,7 +1757,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1828,7 +1828,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1848,7 +1848,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1990,7 +1990,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2000,7 +2000,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2081,7 +2081,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` + `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` UNION ALL SELECT submission_timestamp, @@ -2091,7 +2091,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` + `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2162,7 +2162,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -2172,7 +2172,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:48:45.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:50:59.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.main_v1` + `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-13 21:48:08.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-13 21:59:01.000000000 +0000 @@ -1,7 +1,13 @@ fields: -- type: DATETIME - name: build_hour -- type: INTEGER - name: geckoview_major_version -- type: INTEGER - name: n_pings +- name: build_hour + type: DATETIME + mode: NULLABLE + description: null +- name: geckoview_major_version + type: INTEGER + mode: NULLABLE + description: null +- name: n_pings + type: INTEGER + mode: NULLABLE + description: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/metrics/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/metrics/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/metrics/schema.yaml 2024-06-13 21:48:45.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/metrics/schema.yaml 2024-06-13 21:50:23.000000000 +0000 @@ -1880,8 +1880,7 @@ - name: value type: INTEGER mode: NULLABLE - description: 'Failure occurs when initializing the audio stream. (Migrated from - the geckoview metric of the same name). + description: 'Failure occurs when initializing the audio stream. ' - name: glean_validation_pings_submitted diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fennec_aurora/metrics/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fennec_aurora/metrics/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fennec_aurora/metrics/schema.yaml 2024-06-13 21:48:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fennec_aurora/metrics/schema.yaml 2024-06-13 21:50:25.000000000 +0000 @@ -3265,7 +3265,8 @@ - name: value type: INTEGER mode: NULLABLE - description: 'Failure occurs when initializing the audio stream. + description: 'Failure occurs when initializing the audio stream. (Migrated from + the geckoview metric of the same name). ' - name: glean_validation_pings_submitted diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox/metrics/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox/metrics/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox/metrics/schema.yaml 2024-06-13 21:48:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox/metrics/schema.yaml 2024-06-13 21:50:21.000000000 +0000 @@ -3265,8 +3265,7 @@ - name: value type: INTEGER mode: NULLABLE - description: 'Failure occurs when initializing the audio stream. (Migrated from - the geckoview metric of the same name). + description: 'Failure occurs when initializing the audio stream. ' - name: glean_validation_pings_submitted diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta/metrics/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta/metrics/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta/metrics/schema.yaml 2024-06-13 21:48:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta/metrics/schema.yaml 2024-06-13 21:50:16.000000000 +0000 @@ -3265,7 +3265,8 @@ - name: value type: INTEGER mode: NULLABLE - description: 'Failure occurs when initializing the audio stream. + description: 'Failure occurs when initializing the audio stream. (Migrated from + the geckoview metric of the same name). ' - name: glean_validation_pings_submitted diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:48:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:51:00.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:48:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:51:01.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:48:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:51:01.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:48:45.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:51:01.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:48:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:51:02.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:48:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 21:51:01.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/clients_daily/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/clients_daily/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/clients_daily/schema.yaml 2024-06-13 21:48:44.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/clients_daily/schema.yaml 2024-06-13 21:59:11.000000000 +0000 @@ -5,27 +5,21 @@ - name: submission_date type: DATE mode: NULLABLE - description: null - name: client_id type: STRING mode: NULLABLE - description: null - name: aborts_content_sum type: INTEGER mode: NULLABLE - description: null - name: aborts_gmplugin_sum type: INTEGER mode: NULLABLE - description: null - name: aborts_plugin_sum type: INTEGER mode: NULLABLE - description: null - name: active_addons_count_mean type: FLOAT mode: NULLABLE - description: null - name: active_addons type: RECORD mode: REPEATED @@ -33,92 +27,69 @@ - name: addon_id type: STRING mode: NULLABLE - description: null - name: blocklisted type: BOOLEAN mode: NULLABLE - description: null - name: name type: STRING mode: NULLABLE - description: null - name: user_disabled type: BOOLEAN mode: NULLABLE - description: null - name: app_disabled type: BOOLEAN mode: NULLABLE - description: null - name: version type: STRING mode: NULLABLE - description: null - name: scope type: INTEGER mode: NULLABLE - description: null - name: type type: STRING mode: NULLABLE - description: null - name: foreign_install type: BOOLEAN mode: NULLABLE - description: null - name: has_binary_components type: BOOLEAN mode: NULLABLE - description: null - name: install_day type: INTEGER mode: NULLABLE - description: null - name: update_day type: INTEGER mode: NULLABLE - description: null - name: signed_state type: INTEGER mode: NULLABLE - description: null - name: is_system type: BOOLEAN mode: NULLABLE - description: null - name: is_web_extension type: BOOLEAN mode: NULLABLE - description: null - name: multiprocess_compatible type: BOOLEAN mode: NULLABLE - description: null - description: null - name: active_hours_sum type: FLOAT mode: NULLABLE - description: null - name: addon_compatibility_check_enabled type: BOOLEAN mode: NULLABLE - description: null - name: app_build_id type: STRING mode: NULLABLE - description: null - name: app_display_version type: STRING mode: NULLABLE - description: null - name: app_name type: STRING mode: NULLABLE - description: null - name: app_version type: STRING mode: NULLABLE - description: null - name: attribution type: RECORD mode: NULLABLE @@ -126,19 +97,15 @@ - name: source type: STRING mode: NULLABLE - description: null - name: medium type: STRING mode: NULLABLE - description: null - name: campaign type: STRING mode: NULLABLE - description: null - name: content type: STRING mode: NULLABLE - description: null - name: experiment type: STRING mode: NULLABLE @@ -154,163 +121,123 @@ - name: ua type: STRING mode: NULLABLE - description: null - name: blocklist_enabled type: BOOLEAN mode: NULLABLE - description: null - name: channel type: STRING mode: NULLABLE - description: null - name: client_clock_skew_mean type: FLOAT mode: NULLABLE - description: null - name: client_submission_latency_mean type: FLOAT mode: NULLABLE - description: null - name: cpu_cores type: INTEGER mode: NULLABLE - description: null - name: cpu_count type: INTEGER mode: NULLABLE - description: null - name: cpu_family type: INTEGER mode: NULLABLE - description: null - name: cpu_l2_cache_kb type: INTEGER mode: NULLABLE - description: null - name: cpu_l3_cache_kb type: INTEGER mode: NULLABLE - description: null - name: cpu_model type: INTEGER mode: NULLABLE - description: null - name: cpu_speed_mhz type: INTEGER mode: NULLABLE - description: null - name: cpu_stepping type: INTEGER mode: NULLABLE - description: null - name: cpu_vendor type: STRING mode: NULLABLE - description: null - name: crashes_detected_content_sum type: INTEGER mode: NULLABLE - description: null - name: crashes_detected_gmplugin_sum type: INTEGER mode: NULLABLE - description: null - name: crashes_detected_plugin_sum type: INTEGER mode: NULLABLE - description: null - name: crash_submit_attempt_content_sum type: INTEGER mode: NULLABLE - description: null - name: crash_submit_attempt_main_sum type: INTEGER mode: NULLABLE - description: null - name: crash_submit_attempt_plugin_sum type: INTEGER mode: NULLABLE - description: null - name: crash_submit_success_content_sum type: INTEGER mode: NULLABLE - description: null - name: crash_submit_success_main_sum type: INTEGER mode: NULLABLE - description: null - name: crash_submit_success_plugin_sum type: INTEGER mode: NULLABLE - description: null - name: default_search_engine type: STRING mode: NULLABLE - description: null - name: default_search_engine_data_load_path type: STRING mode: NULLABLE - description: null - name: default_search_engine_data_name type: STRING mode: NULLABLE - description: null - name: default_search_engine_data_origin type: STRING mode: NULLABLE - description: null - name: default_search_engine_data_submission_url type: STRING mode: NULLABLE - description: null - name: devtools_toolbox_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: distribution_id type: STRING mode: NULLABLE - description: null - name: e10s_enabled type: BOOLEAN mode: NULLABLE - description: null - name: env_build_arch type: STRING mode: NULLABLE - description: null - name: env_build_id type: STRING mode: NULLABLE - description: null - name: env_build_version type: STRING mode: NULLABLE - description: null - name: environment_settings_intl_accept_languages type: STRING mode: REPEATED - description: null - name: environment_settings_intl_app_locales type: STRING mode: REPEATED - description: null - name: environment_settings_intl_available_locales type: STRING mode: REPEATED - description: null - name: environment_settings_intl_requested_locales type: STRING mode: REPEATED - description: null - name: environment_settings_intl_system_locales type: STRING mode: REPEATED - description: null - name: environment_settings_intl_regional_prefs_locales type: STRING mode: REPEATED - description: null - name: experiments type: RECORD mode: REPEATED @@ -318,352 +245,264 @@ - name: key type: STRING mode: NULLABLE - description: null - name: value type: STRING mode: NULLABLE - description: null - description: null - name: first_paint_mean type: FLOAT mode: NULLABLE - description: null - name: flash_version type: STRING mode: NULLABLE - description: null - name: country type: STRING mode: NULLABLE - description: null - name: city type: STRING mode: NULLABLE - description: null - name: geo_subdivision1 type: STRING mode: NULLABLE - description: null - name: geo_subdivision2 type: STRING mode: NULLABLE - description: null - name: isp_name type: STRING mode: NULLABLE - description: null - name: isp_organization type: STRING mode: NULLABLE - description: null - name: gfx_features_advanced_layers_status type: STRING mode: NULLABLE - description: null - name: gfx_features_d2d_status type: STRING mode: NULLABLE - description: null - name: gfx_features_d3d11_status type: STRING mode: NULLABLE - description: null - name: gfx_features_gpu_process_status type: STRING mode: NULLABLE - description: null - name: histogram_parent_devtools_aboutdebugging_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_animationinspector_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_browserconsole_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_canvasdebugger_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_computedview_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_custom_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_dom_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_eyedropper_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_fontinspector_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_inspector_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_jsbrowserdebugger_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_jsdebugger_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_jsprofiler_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_layoutview_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_memory_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_menu_eyedropper_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_netmonitor_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_options_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_paintflashing_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_picker_eyedropper_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_responsive_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_ruleview_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_scratchpad_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_scratchpad_window_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_shadereditor_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_storage_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_styleeditor_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_webaudioeditor_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_webconsole_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_webide_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: install_year type: INTEGER mode: NULLABLE - description: null - name: is_default_browser type: BOOLEAN mode: NULLABLE - description: null - name: is_wow64 type: BOOLEAN mode: NULLABLE - description: null - name: locale type: STRING mode: NULLABLE - description: null - name: memory_mb type: INTEGER mode: NULLABLE - description: null - name: normalized_channel type: STRING mode: NULLABLE - description: null - name: normalized_os_version type: STRING mode: NULLABLE - description: null - name: os type: STRING mode: NULLABLE - description: null - name: os_service_pack_major type: INTEGER mode: NULLABLE - description: null - name: os_service_pack_minor type: INTEGER mode: NULLABLE - description: null - name: os_version type: STRING mode: NULLABLE - description: null - name: pings_aggregated_by_this_row type: INTEGER mode: NULLABLE - description: null - name: places_bookmarks_count_mean type: FLOAT mode: NULLABLE - description: null - name: places_pages_count_mean type: FLOAT mode: NULLABLE - description: null - name: plugin_hangs_sum type: INTEGER mode: NULLABLE - description: null - name: plugins_infobar_allow_sum type: INTEGER mode: NULLABLE - description: null - name: plugins_infobar_block_sum type: INTEGER mode: NULLABLE - description: null - name: plugins_infobar_shown_sum type: INTEGER mode: NULLABLE - description: null - name: plugins_notification_shown_sum type: INTEGER mode: NULLABLE - description: null - name: previous_bui ```

⚠️ Only part of the diff is displayed.

Link to full diff

dataops-ci-bot commented 2 weeks ago

Integration report for "test"

sql.diff

Click to expand! ```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare.py /tmp/workspace/generated-sql/dags/bqetl_cloudflare.py --- /tmp/workspace/main-generated-sql/dags/bqetl_cloudflare.py 2024-06-13 23:39:38.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_cloudflare.py 2024-06-13 23:55:32.000000000 +0000 @@ -84,20 +84,3 @@ owner="kwindau@mozilla.com", email=["kwindau@mozilla.com"], ) - - cloudflare_derived__os_usage__v1 = GKEPodOperator( - task_id="cloudflare_derived__os_usage__v1", - arguments=[ - "python", - "sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py", - ] - + [ - "--date", - "{{ds}}", - "--cloudflare_api_token", - "{{ var.value.cloudflare_auth_token}}", - ], - image="gcr.io/moz-fx-data-airflow-prod-88e0/bigquery-etl:latest", - owner="kwindau@mozilla.com", - email=["kwindau@mozilla.com"], - ) Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1: query.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:36:33.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:38:33.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.events_v1` + `moz-fx-data-shared-prod.bedrock_live.interaction_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml 2024-06-13 23:37:34.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/metadata.yaml 2024-06-13 23:52:26.000000000 +0000 @@ -6,15 +6,6 @@ labels: incremental: true owner1: kwindau - dag: bqetl_cloudflare -scheduling: - dag_name: bqetl_cloudflare - date_partition_parameter: dte - arguments: - - --date - - '{{ds}}' - - --cloudflare_api_token - - '{{ var.value.cloudflare_auth_token}}' bigquery: time_partitioning: type: day diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py 2024-06-13 23:35:56.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/cloudflare_derived/os_usage_v1/query.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,385 +0,0 @@ -# Load libraries -import json -from datetime import datetime, timedelta -import pandas as pd -import requests -from argparse import ArgumentParser -from google.cloud import bigquery -from google.cloud import storage - -# Configurations -os_usg_configs = { - "timeout_limit": 2000, - "device_types": ["DESKTOP", "MOBILE", "OTHER", "ALL"], - "locations": [ - "ALL", - "BE", - "BG", - "CA", - "CZ", - "DE", - "DK", - "EE", - "ES", - "FI", - "FR", - "GB", - "HR", - "IE", - "IT", - "CY", - "LV", - "LT", - "LU", - "HU", - "MT", - "MX", - "NL", - "AT", - "PL", - "PT", - "RO", - "SI", - "SK", - "US", - "SE", - "GR", - ], - "bucket": "gs://moz-fx-data-prod-external-data/", - "bucket_no_gs": "moz-fx-data-prod-external-data", - "results_stg_gcs_fpth": "cloudflare/os_usage/RESULTS_STAGING/%s_results_from_%s_run.csv", - "results_archive_gcs_fpth": "cloudflare/os_usage/RESULTS_ARCHIVE/%s_results_from_%s_run.csv", - "errors_stg_gcs_fpth": "cloudflare/os_usage/ERRORS_STAGING/%s_results_from_%s_run.csv", - "errors_archive_gcs_fpth": "cloudflare/os_usage/ERRORS_ARCHIVE/%s_results_from_%s_run.csv", - "gcp_project_id": "moz-fx-data-shared-prod", - "gcp_conn_id": "google_cloud_shared_prod", - "results_bq_stg_table": "moz-fx-data-shared-prod.cloudflare_derived.os_results_stg", - "errors_bq_stg_table": "moz-fx-data-shared-prod.cloudflare_derived.os_errors_stg", -} - - -# Define a function to move a GCS object then delete the original -def move_blob(bucket_name, blob_name, destination_bucket_name, destination_blob_name): - """Moves a blob from one bucket to another with a new name.""" - storage_client = storage.Client() - source_bucket = storage_client.bucket(bucket_name) - source_blob = source_bucket.blob(blob_name) - destination_bucket = storage_client.bucket(destination_bucket_name) - destination_generation_match_precondition = 0 - - blob_copy = source_bucket.copy_blob( - source_blob, - destination_bucket, - destination_blob_name, - if_generation_match=destination_generation_match_precondition, - ) - source_bucket.delete_blob(blob_name) - - print( - "Blob {} in bucket {} moved to blob {} in bucket {}.".format( - source_blob.name, - source_bucket.name, - blob_copy.name, - destination_bucket.name, - ) - ) - - -# Function to configure the API URL -def generate_os_timeseries_api_call(strt_dt, end_dt, agg_int, location, device_type): - """Generate the API call for Operating System Usage Data.""" - if location == "ALL" and device_type == "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&format=json&aggInterval={agg_int}" - elif location != "ALL" and device_type == "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={strt_dt}T00:00:00.000Z&location={location}&format=json&aggInterval={agg_int}" - elif location == "ALL" and device_type != "ALL": - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&deviceType={device_type}&format=json&aggInterval={agg_int}" - else: - os_usage_api_url = f"https://api.cloudflare.com/client/v4/radar/http/timeseries_groups/os?dateStart={strt_dt}T00:00:00.000Z&dateEnd={end_dt}T00:00:00.000Z&location={location}&deviceType={device_type}&format=json&aggInterval={agg_int}" - return os_usage_api_url - - -def get_os_usage_data(date_of_interest, auth_token): - """Pull OS usage data from the Cloudflare API and save errors & results to GCS.""" - # Calculate start date and end date - logical_dag_dt = date_of_interest - logical_dag_dt_as_date = datetime.strptime(logical_dag_dt, "%Y-%m-%d").date() - start_date = logical_dag_dt_as_date - timedelta(days=4) - end_date = start_date + timedelta(days=1) - print("Start Date: ", start_date) - print("End Date: ", end_date) - - # Configure request headers - bearer_string = f"Bearer {auth_token}" - headers = {"Authorization": bearer_string} - - # Initialize the empty results & errors dataframe - result_df = pd.DataFrame( - { - "Timestamps": [], - "OS": [], - "Location": [], - "DeviceType": [], - "Share": [], - "ConfidenceLevel": [], - "AggrInterval": [], - "Normalization": [], - "LastUpdatedTS": [], - } - ) - - # Initialize an errors dataframe - errors_df = pd.DataFrame( - {"StartTime": [], "EndTime": [], "Location": [], "DeviceType": []} - ) - - # Go through all combinations, submit API requests - for device_type in os_usg_configs["device_types"]: - for loc in os_usg_configs["locations"]: - print("Device Type: ", device_type) - print("Loc: ", loc) - - # Generate the URL with given parameters - os_usage_api_url = generate_os_timeseries_api_call( - start_date, end_date, "1d", loc, device_type - ) - try: - # Call the API and save the response as JSON - response = requests.get( - os_usage_api_url, - headers=headers, - timeout=os_usg_configs["timeout_limit"], - ) - response_json = json.loads(response.text) - - # If response was successful, get the result - if response_json["success"] is True: - result = response_json["result"] - # Parse metadata - conf_lvl = result["meta"]["confidenceInfo"]["level"] - aggr_intvl = result["meta"]["aggInterval"] - nrmlztn = result["meta"]["normalization"] - lst_upd = result["meta"]["lastUpdated"] - data_dict = result["serie_0"] - - for key, val in data_dict.items(): - if key != 'timestamps': - new_result_df = pd.DataFrame( - { - "Timestamps": data_dict["timestamps"], - "OS": [key] * len(val), - "Location": [loc] * len(val), - "DeviceType": [device_type] * len(val), - "Share": val, - "ConfidenceLevel": [conf_lvl] * len(val), - "AggrInterval": [aggr_intvl] * len(val), - "Normalization": [nrmlztn] * len(val), - "LastUpdatedTS": [lst_upd] * len(val), - } - ) - result_df = pd.concat([result_df, new_result_df]) - - # If response was not successful, get the errors - else: - # errors = response_json["errors"] # Maybe add to capture, right now not using this - new_errors_df = pd.DataFrame( - { - "StartTime": [start_date], - "EndTime": [end_date], - "Location": [loc], - "DeviceType": [device_type], - } - ) - errors_df = pd.concat([errors_df, new_errors_df]) - except: - new_errors_df = pd.DataFrame( - { - "StartTime": [start_date], - "EndTime": [end_date], - "Location": [loc], - "DeviceType": [device_type], - } - ) - errors_df = pd.concat([errors_df, new_errors_df]) - - result_fpath = os_usg_configs["bucket"] + os_usg_configs["results_stg_gcs_fpth"] % ( - start_date, - logical_dag_dt, - ) - errors_fpath = os_usg_configs["bucket"] + os_usg_configs["errors_stg_gcs_fpth"] % ( - start_date, - logical_dag_dt, - ) - - result_df.to_csv(result_fpath, index=False) - errors_df.to_csv(errors_fpath, index=False) - print("Wrote errors to: ", errors_fpath) - print("Wrote results to: ", result_fpath) - - # Write a summary to the logs - len_results = str(len(result_df)) - len_errors = str(len(errors_df)) - result_summary = f"# Result Rows: {len_results}; # of Error Rows: {len_errors}" - return result_summary - - -def main(): - """Call the API, save data to GCS, load to BQ staging, delete & load to BQ gold""" - parser = ArgumentParser(description=__doc__) - parser.add_argument("--date", required=True) - parser.add_argument("--cloudflare_api_token", required=True) - parser.add_argument("--project", default="moz-fx-data-shared-prod") - parser.add_argument("--dataset", default="cloudflare_derived") - - args = parser.parse_args() - print("Running for date: ") - print(args.date) - - # STEP 1 - Pull the data from the API, save results & errors to GCS staging area - result_summary = get_os_usage_data(args.date, args.cloudflare_api_token) - print("result_summary") - print(result_summary) - - # Create a bigquery client - client = bigquery.Client(args.project) - - result_uri = os_usg_configs["bucket"] + os_usg_configs["results_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - error_uri = os_usg_configs["bucket"] + os_usg_configs["errors_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - print("result_uri") - print(result_uri) - - print("error_uri") - print(error_uri) - - # STEP 2 - Copy the result data from GCS staging to BQ staging table - load_result_csv_to_bq_stg_job = client.load_table_from_uri( - result_uri, - os_usg_configs["results_bq_stg_table"], - job_config=bigquery.LoadJobConfig( - create_disposition="CREATE_IF_NEEDED", - write_disposition="WRITE_TRUNCATE", - schema=[ - {"name": "Timestamps", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "OS", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Location", "type": "STRING", "mode": "NULLABLE"}, - {"name": "DeviceType", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Share", "type": "NUMERIC", "mode": "NULLABLE"}, - {"name": "ConfidenceLevel", "type": "STRING", "mode": "NULLABLE"}, - {"name": "AggrInterval", "type": "STRING", "mode": "NULLABLE"}, - {"name": "Normalization", "type": "STRING", "mode": "NULLABLE"}, - {"name": "LastUpdatedTS", "type": "TIMESTAMP", "mode": "NULLABLE"}, - ], - skip_leading_rows=1, - source_format=bigquery.SourceFormat.CSV, - ), - ) - - load_result_csv_to_bq_stg_job.result() - result_bq_stg_tbl = client.get_table(os_usg_configs["results_bq_stg_table"]) - print("Loaded {} rows to results staging.".format(result_bq_stg_tbl.num_rows)) - - # STEP 3 - Copy the error data from GCS staging to BQ staging table - load_error_csv_to_bq_stg_job = client.load_table_from_uri( - error_uri, - os_usg_configs["errors_bq_stg_table"], - job_config=bigquery.LoadJobConfig( - create_disposition="CREATE_IF_NEEDED", - write_disposition="WRITE_TRUNCATE", - schema=[ - {"name": "StartTime", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "EndTime", "type": "TIMESTAMP", "mode": "REQUIRED"}, - {"name": "Location", "type": "STRING", "mode": "NULLABLE"}, - {"name": "DeviceType", "type": "STRING", "mode": "NULLABLE"}, - ], - skip_leading_rows=1, - source_format=bigquery.SourceFormat.CSV, - ), - ) - - load_error_csv_to_bq_stg_job.result() - error_bq_stg_tbl = client.get_table(os_usg_configs["errors_bq_stg_table"]) - print("Loaded {} rows to errors staging.".format(error_bq_stg_tbl.num_rows)) - - # STEP 4 - Delete results from gold for this day, if there are any already (so if rerun, no issues will occur) - del_exstng_gold_res_for_date = f"""DELETE FROM `moz-fx-data-shared-prod.cloudflare_derived.os_usage_v1` WHERE dte = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - del_gold_res_job = client.query(del_exstng_gold_res_for_date) - del_gold_res_job.result() - print("Deleted anything already existing for this date from results gold") - - # STEP 5 - Delete errors from gold for this day, if there are any already (so if rerun, no issues will occur) - del_exstng_gold_err_for_date = f"""DELETE FROM `moz-fx-data-shared-prod.cloudflare_derived.os_usage_errors_v1` WHERE dte = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - del_gold_err_job = client.query(del_exstng_gold_err_for_date) - del_gold_err_job.result() - print("Deleted anything already existing for this date from errors gold") - - # STEP 6 - Load results from stage to gold # NEED TO UPDATE THIS STILL - device_usg_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_v1` -SELECT -CAST(Timestamps AS date) AS dte, -OS AS os, -Location AS location, -DeviceType AS device_type, -Share AS os_share, -Normalization AS normalization_type, -LastUpdatedTS AS last_updated_ts -FROM `moz-fx-data-shared-prod.cloudflare_derived.os_results_stg` -WHERE CAST(Timestamps as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - load_res_to_gold = client.query(device_usg_stg_to_gold_query) - load_res_to_gold.result() - - # STEP 7 - Load errors from stage to gold - browser_usg_errors_stg_to_gold_query = f""" INSERT INTO `moz-fx-data-shared-prod.cloudflare_derived.os_usage_errors_v1` -SELECT -CAST(StartTime as date) AS dte, -Location AS location, -DeviceType AS device_type -FROM `moz-fx-data-shared-prod.cloudflare_derived.os_errors_stg` -WHERE CAST(StartTime as date) = DATE_SUB('{args.date}', INTERVAL 4 DAY) """ - load_err_to_gold = client.query(browser_usg_errors_stg_to_gold_query) - load_err_to_gold.result() - - # Initialize a storage client to use in next steps - storage_client = storage.Client() - - # STEP 8 - Copy the result CSV from stage to archive, then delete from stage - # Calculate the fpaths we will use ahead of time - result_stg_fpath = os_usg_configs["results_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - result_archive_fpath = os_usg_configs["results_archive_gcs_fpath"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - move_blob( - os_usg_configs["bucket_no_gs"], - result_stg_fpath, - os_usg_configs["bucket_no_gs"], - result_archive_fpath, - ) - - # STEP 9 - Copy the error CSV from stage to archive, then delete from stage - error_stg_fpath = os_usg_configs["errors_stg_gcs_fpth"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - error_archive_fpath = os_usg_configs["errors_archive_gcs_fpath"] % ( - datetime.strptime(args.date, "%Y-%m-%d").date() - timedelta(days=4), - args.date, - ) - move_blob( - os_usg_configs["bucket_no_gs"], - error_stg_fpath, - os_usg_configs["bucket_no_gs"], - error_archive_fpath, - ) - - -if __name__ == "__main__": - main() diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-13 23:35:56.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-13 23:45:32.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: source + mode: NULLABLE +- name: source type: STRING -- mode: NULLABLE - name: event_type + mode: NULLABLE +- name: event_type type: STRING -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: subdivision1 + mode: NULLABLE +- name: subdivision1 type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: event_count + mode: NULLABLE +- name: event_count type: INTEGER -- mode: NULLABLE - name: user_count + mode: NULLABLE +- name: user_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-13 23:35:56.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-13 23:45:32.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: impression_count + mode: NULLABLE +- name: impression_count type: INTEGER -- mode: NULLABLE - name: click_count + mode: NULLABLE +- name: click_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-13 23:35:56.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-13 23:44:18.000000000 +0000 @@ -26,6 +26,9 @@ - name: adjust_network type: STRING mode: NULLABLE +- name: install_source + type: STRING + mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-13 23:35:56.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-13 23:44:18.000000000 +0000 @@ -48,6 +48,10 @@ description: 'The type of source of a client installation. ' +- name: install_source + type: STRING + mode: NULLABLE + description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/use_counters/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/use_counters/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/use_counters/schema.yaml 2024-06-13 23:36:33.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/use_counters/schema.yaml 2024-06-13 23:45:19.000000000 +0000 @@ -1,11 +1,11 @@ fields: - name: normalized_app_id - mode: NULLABLE type: STRING + mode: NULLABLE description: App ID of the channel data was received from - name: normalized_channel - mode: NULLABLE type: STRING + mode: NULLABLE description: Normalized channel name - name: additional_properties type: STRING diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:36:33.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:38:34.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -80,7 +80,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-13 23:36:33.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-13 23:40:09.000000000 +0000 @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -75,7 +75,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -572,7 +572,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -582,7 +582,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -592,7 +592,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -663,7 +663,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -673,7 +673,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -683,7 +683,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -754,7 +754,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -764,7 +764,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -774,7 +774,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1555,7 +1555,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1565,7 +1565,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1575,7 +1575,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1646,7 +1646,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1656,7 +1656,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1666,7 +1666,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1737,7 +1737,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1747,7 +1747,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1757,7 +1757,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1828,7 +1828,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1838,7 +1838,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1848,7 +1848,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1990,7 +1990,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2000,7 +2000,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -2010,7 +2010,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.events_v1` + `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:36:33.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:38:33.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.main_v1` + `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-13 23:35:56.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-13 23:45:42.000000000 +0000 @@ -1,7 +1,13 @@ fields: -- type: DATETIME - name: build_hour -- type: INTEGER - name: geckoview_major_version -- type: INTEGER - name: n_pings +- name: build_hour + type: DATETIME + mode: NULLABLE + description: null +- name: geckoview_major_version + type: INTEGER + mode: NULLABLE + description: null +- name: n_pings + type: INTEGER + mode: NULLABLE + description: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/metrics/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/metrics/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/metrics/schema.yaml 2024-06-13 23:36:33.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/metrics/schema.yaml 2024-06-13 23:38:00.000000000 +0000 @@ -1880,8 +1880,7 @@ - name: value type: INTEGER mode: NULLABLE - description: 'Failure occurs when initializing the audio stream. (Migrated from - the geckoview metric of the same name). + description: 'Failure occurs when initializing the audio stream. ' - name: glean_validation_pings_submitted diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fennec_aurora/metrics/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fennec_aurora/metrics/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fennec_aurora/metrics/schema.yaml 2024-06-13 23:36:33.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fennec_aurora/metrics/schema.yaml 2024-06-13 23:38:03.000000000 +0000 @@ -3265,7 +3265,8 @@ - name: value type: INTEGER mode: NULLABLE - description: 'Failure occurs when initializing the audio stream. + description: 'Failure occurs when initializing the audio stream. (Migrated from + the geckoview metric of the same name). ' - name: glean_validation_pings_submitted diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox/metrics/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox/metrics/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox/metrics/schema.yaml 2024-06-13 23:36:34.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox/metrics/schema.yaml 2024-06-13 23:38:02.000000000 +0000 @@ -3265,8 +3265,7 @@ - name: value type: INTEGER mode: NULLABLE - description: 'Failure occurs when initializing the audio stream. (Migrated from - the geckoview metric of the same name). + description: 'Failure occurs when initializing the audio stream. ' - name: glean_validation_pings_submitted diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta/metrics/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta/metrics/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta/metrics/schema.yaml 2024-06-13 23:36:34.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_beta/metrics/schema.yaml 2024-06-13 23:37:56.000000000 +0000 @@ -3265,7 +3265,8 @@ - name: value type: INTEGER mode: NULLABLE - description: 'Failure occurs when initializing the audio stream. + description: 'Failure occurs when initializing the audio stream. (Migrated from + the geckoview metric of the same name). ' - name: glean_validation_pings_submitted diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:36:34.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:38:35.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:36:34.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:38:35.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:36:33.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:38:36.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:36:34.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:38:36.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:36:33.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:38:36.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:36:34.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-13 23:38:36.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/clients_daily/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/clients_daily/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/clients_daily/schema.yaml 2024-06-13 23:36:34.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/clients_daily/schema.yaml 2024-06-13 23:45:51.000000000 +0000 @@ -5,27 +5,21 @@ - name: submission_date type: DATE mode: NULLABLE - description: null - name: client_id type: STRING mode: NULLABLE - description: null - name: aborts_content_sum type: INTEGER mode: NULLABLE - description: null - name: aborts_gmplugin_sum type: INTEGER mode: NULLABLE - description: null - name: aborts_plugin_sum type: INTEGER mode: NULLABLE - description: null - name: active_addons_count_mean type: FLOAT mode: NULLABLE - description: null - name: active_addons type: RECORD mode: REPEATED @@ -33,92 +27,69 @@ - name: addon_id type: STRING mode: NULLABLE - description: null - name: blocklisted type: BOOLEAN mode: NULLABLE - description: null - name: name type: STRING mode: NULLABLE - description: null - name: user_disabled type: BOOLEAN mode: NULLABLE - description: null - name: app_disabled type: BOOLEAN mode: NULLABLE - description: null - name: version type: STRING mode: NULLABLE - description: null - name: scope type: INTEGER mode: NULLABLE - description: null - name: type type: STRING mode: NULLABLE - description: null - name: foreign_install type: BOOLEAN mode: NULLABLE - description: null - name: has_binary_components type: BOOLEAN mode: NULLABLE - description: null - name: install_day type: INTEGER mode: NULLABLE - description: null - name: update_day type: INTEGER mode: NULLABLE - description: null - name: signed_state type: INTEGER mode: NULLABLE - description: null - name: is_system type: BOOLEAN mode: NULLABLE - description: null - name: is_web_extension type: BOOLEAN mode: NULLABLE - description: null - name: multiprocess_compatible type: BOOLEAN mode: NULLABLE - description: null - description: null - name: active_hours_sum type: FLOAT mode: NULLABLE - description: null - name: addon_compatibility_check_enabled type: BOOLEAN mode: NULLABLE - description: null - name: app_build_id type: STRING mode: NULLABLE - description: null - name: app_display_version type: STRING mode: NULLABLE - description: null - name: app_name type: STRING mode: NULLABLE - description: null - name: app_version type: STRING mode: NULLABLE - description: null - name: attribution type: RECORD mode: NULLABLE @@ -126,19 +97,15 @@ - name: source type: STRING mode: NULLABLE - description: null - name: medium type: STRING mode: NULLABLE - description: null - name: campaign type: STRING mode: NULLABLE - description: null - name: content type: STRING mode: NULLABLE - description: null - name: experiment type: STRING mode: NULLABLE @@ -154,163 +121,123 @@ - name: ua type: STRING mode: NULLABLE - description: null - name: blocklist_enabled type: BOOLEAN mode: NULLABLE - description: null - name: channel type: STRING mode: NULLABLE - description: null - name: client_clock_skew_mean type: FLOAT mode: NULLABLE - description: null - name: client_submission_latency_mean type: FLOAT mode: NULLABLE - description: null - name: cpu_cores type: INTEGER mode: NULLABLE - description: null - name: cpu_count type: INTEGER mode: NULLABLE - description: null - name: cpu_family type: INTEGER mode: NULLABLE - description: null - name: cpu_l2_cache_kb type: INTEGER mode: NULLABLE - description: null - name: cpu_l3_cache_kb type: INTEGER mode: NULLABLE - description: null - name: cpu_model type: INTEGER mode: NULLABLE - description: null - name: cpu_speed_mhz type: INTEGER mode: NULLABLE - description: null - name: cpu_stepping type: INTEGER mode: NULLABLE - description: null - name: cpu_vendor type: STRING mode: NULLABLE - description: null - name: crashes_detected_content_sum type: INTEGER mode: NULLABLE - description: null - name: crashes_detected_gmplugin_sum type: INTEGER mode: NULLABLE - description: null - name: crashes_detected_plugin_sum type: INTEGER mode: NULLABLE - description: null - name: crash_submit_attempt_content_sum type: INTEGER mode: NULLABLE - description: null - name: crash_submit_attempt_main_sum type: INTEGER mode: NULLABLE - description: null - name: crash_submit_attempt_plugin_sum type: INTEGER mode: NULLABLE - description: null - name: crash_submit_success_content_sum type: INTEGER mode: NULLABLE - description: null - name: crash_submit_success_main_sum type: INTEGER mode: NULLABLE - description: null - name: crash_submit_success_plugin_sum type: INTEGER mode: NULLABLE - description: null - name: default_search_engine type: STRING mode: NULLABLE - description: null - name: default_search_engine_data_load_path type: STRING mode: NULLABLE - description: null - name: default_search_engine_data_name type: STRING mode: NULLABLE - description: null - name: default_search_engine_data_origin type: STRING mode: NULLABLE - description: null - name: default_search_engine_data_submission_url type: STRING mode: NULLABLE - description: null - name: devtools_toolbox_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: distribution_id type: STRING mode: NULLABLE - description: null - name: e10s_enabled type: BOOLEAN mode: NULLABLE - description: null - name: env_build_arch type: STRING mode: NULLABLE - description: null - name: env_build_id type: STRING mode: NULLABLE - description: null - name: env_build_version type: STRING mode: NULLABLE - description: null - name: environment_settings_intl_accept_languages type: STRING mode: REPEATED - description: null - name: environment_settings_intl_app_locales type: STRING mode: REPEATED - description: null - name: environment_settings_intl_available_locales type: STRING mode: REPEATED - description: null - name: environment_settings_intl_requested_locales type: STRING mode: REPEATED - description: null - name: environment_settings_intl_system_locales type: STRING mode: REPEATED - description: null - name: environment_settings_intl_regional_prefs_locales type: STRING mode: REPEATED - description: null - name: experiments type: RECORD mode: REPEATED @@ -318,352 +245,264 @@ - name: key type: STRING mode: NULLABLE - description: null - name: value type: STRING mode: NULLABLE - description: null - description: null - name: first_paint_mean type: FLOAT mode: NULLABLE - description: null - name: flash_version type: STRING mode: NULLABLE - description: null - name: country type: STRING mode: NULLABLE - description: null - name: city type: STRING mode: NULLABLE - description: null - name: geo_subdivision1 type: STRING mode: NULLABLE - description: null - name: geo_subdivision2 type: STRING mode: NULLABLE - description: null - name: isp_name type: STRING mode: NULLABLE - description: null - name: isp_organization type: STRING mode: NULLABLE - description: null - name: gfx_features_advanced_layers_status type: STRING mode: NULLABLE - description: null - name: gfx_features_d2d_status type: STRING mode: NULLABLE - description: null - name: gfx_features_d3d11_status type: STRING mode: NULLABLE - description: null - name: gfx_features_gpu_process_status type: STRING mode: NULLABLE - description: null - name: histogram_parent_devtools_aboutdebugging_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_animationinspector_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_browserconsole_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_canvasdebugger_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_computedview_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_custom_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_dom_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_eyedropper_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_fontinspector_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_inspector_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_jsbrowserdebugger_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_jsdebugger_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_jsprofiler_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_layoutview_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_memory_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_menu_eyedropper_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_netmonitor_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_options_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_paintflashing_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_picker_eyedropper_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_responsive_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_ruleview_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_scratchpad_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_scratchpad_window_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_shadereditor_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_storage_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_styleeditor_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_webaudioeditor_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_webconsole_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: histogram_parent_devtools_webide_opened_count_sum type: INTEGER mode: NULLABLE - description: null - name: install_year type: INTEGER mode: NULLABLE - description: null - name: is_default_browser type: BOOLEAN mode: NULLABLE - description: null - name: is_wow64 type: BOOLEAN mode: NULLABLE - description: null - name: locale type: STRING mode: NULLABLE - description: null - name: memory_mb type: INTEGER mode: NULLABLE - descri ```

⚠️ Only part of the diff is displayed.

Link to full diff

dataops-ci-bot commented 2 weeks ago

Integration report for "Merge branch 'main' into newtab_table_defaultUI_pocketID"

sql.diff

Click to expand! ```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql 2024-06-17 12:41:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql 2024-06-17 12:41:40.000000000 +0000 @@ -20,16 +20,6 @@ WHERE DATE(submission_timestamp) = @submission_date AND category IN ('newtab', 'topsites', 'newtab.search', 'newtab.search.ad', 'pocket') - AND name IN ( - 'closed', - 'opened', - 'impression', - 'issued', - 'click', - 'save', - 'topic_click', - 'dismiss' - ) ), visit_metadata AS ( SELECT @@ -54,6 +44,7 @@ ANY_VALUE(metrics.string.newtab_homepage_category) AS newtab_homepage_category, ANY_VALUE(metrics.string.newtab_newtab_category) AS newtab_newtab_category, ANY_VALUE(metrics.boolean.newtab_search_enabled) AS newtab_search_enabled, + ANY_VALUE(metrics.boolean.newtab_weather_enabled) AS newtab_weather_widget_enabled, ANY_VALUE(metrics.quantity.topsites_rows) AS topsites_rows, ANY_VALUE(metrics.string_list.newtab_blocked_sponsors) AS newtab_blocked_sponsors, ANY_VALUE(ping_info.experiments) AS experiments, @@ -110,6 +101,7 @@ events_unnested WHERE event_category IN ('newtab.search', 'newtab.search.ad') + AND event_name IN ('click', 'impression', 'issued') GROUP BY newtab_visit_id, search_engine, @@ -180,6 +172,7 @@ ) WHERE event_category = 'topsites' + AND event_name IN ('dismiss', 'click', 'impression') GROUP BY newtab_visit_id, topsite_tile_position, @@ -218,6 +211,8 @@ SELECT mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS pocket_story_position, + mozfun.map.get_key(event_details, "tile_id") AS pocket_tile_id, + mozfun.map.get_key(event_details, "recommendation_id") AS pocket_recommendation_id, COUNTIF(event_name = 'save') AS pocket_saves, COUNTIF(event_name = 'click') AS pocket_clicks, COUNTIF(event_name = 'impression') AS pocket_impressions, @@ -249,9 +244,12 @@ events_unnested WHERE event_category = 'pocket' + AND event_name IN ('impression', 'click', 'save') GROUP BY newtab_visit_id, - pocket_story_position + pocket_story_position, + pocket_tile_id, + pocket_recommendation_id ), pocket_summary AS ( SELECT @@ -259,6 +257,8 @@ ARRAY_AGG( STRUCT( pocket_story_position, + pocket_tile_id, + pocket_recommendation_id, pocket_impressions, sponsored_pocket_impressions, organic_pocket_impressions, @@ -275,6 +275,99 @@ GROUP BY newtab_visit_id ), +wallpaper_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + mozfun.map.get_key(event_details, "selected_wallpaper") AS wallpaper_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_click') AS wallpaper_clicks, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "true" + ) AS wallpaper_clicks_had_previous_wallpaper, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "false" + ) AS wallpaper_clicks_first_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_category_click') AS wallpaper_category_clicks, + COUNTIF(event_name = 'wallpaper_highlight_dismissed') AS wallpaper_highlight_dismissals, + COUNTIF(event_name = 'wallpaper_highlight_cta_click') AS wallpaper_highlight_cta_clicks + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'wallpaper_click', + 'wallpaper_category_click', + 'wallpaper_highlight_cta_clicks', + 'wallpaper_highlight_dismissed' + ) + GROUP BY + newtab_visit_id, + wallpaper_selected_wallpaper +), +wallpaper_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + wallpaper_selected_wallpaper, + wallpaper_clicks, + wallpaper_clicks_had_previous_wallpaper, + wallpaper_clicks_first_selected_wallpaper, + wallpaper_category_clicks, + wallpaper_highlight_dismissals, + wallpaper_highlight_cta_clicks + ) + ) AS wallpaper_interactions + FROM + wallpaper_events + GROUP BY + newtab_visit_id +), +weather_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + COUNTIF(event_name = 'weather_impression') AS weather_widget_impressions, + COUNTIF(event_name = 'weather_open_provider_url') AS weather_widget_clicks, + COUNTIF(event_name = 'weather_load_error') AS weather_widget_load_errors, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "detailed" + ) AS weather_widget_change_display_to_detailed, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "simple" + ) AS weather_widget_change_display_to_simple + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'weather_impression', + 'weather_open_provider_url', + 'weather_load_error', + 'weather_change_display' + ) + GROUP BY + newtab_visit_id +), +weather_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + weather_widget_impressions, + weather_widget_clicks, + weather_widget_load_errors, + weather_widget_change_display_to_detailed, + weather_widget_change_display_to_simple + ) + ) AS weather_interactions + FROM + weather_events + GROUP BY + newtab_visit_id +), combined_newtab_activity AS ( SELECT * @@ -289,6 +382,12 @@ LEFT JOIN pocket_summary USING (newtab_visit_id) + LEFT JOIN + wallpaper_summary + USING (newtab_visit_id) + LEFT JOIN + weather_summary + USING (newtab_visit_id) WHERE -- Keep only rows with interactions, unless we receive a valid newtab.opened event. -- This is meant to drop only interactions that only have a newtab.closed event on the same partition @@ -297,6 +396,8 @@ OR search_interactions IS NOT NULL OR topsite_tile_interactions IS NOT NULL OR pocket_interactions IS NOT NULL + OR wallpaper_interactions IS NOT NULL + OR weather_interactions IS NOT NULL ), client_profile_info AS ( SELECT @@ -311,7 +412,15 @@ client_id ) SELECT - * + *, + CASE + WHEN ( + (newtab_open_source = "about:home" AND newtab_homepage_category = "enabled") + OR (newtab_open_source = "about:newtab" AND newtab_newtab_category = "enabled") + ) + THEN "default" + ELSE "non-default" + END AS newtab_default_ui, FROM combined_newtab_activity LEFT JOIN diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml 2024-06-17 12:41:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml 2024-06-17 12:41:40.000000000 +0000 @@ -215,6 +215,12 @@ - mode: NULLABLE name: organic_pocket_saves type: INTEGER + - name: pocket_tile_id + type: STRING + mode: NULLABLE + - name: pocket_recommendation_id + type: STRING + mode: NULLABLE mode: REPEATED name: pocket_interactions type: RECORD @@ -224,3 +230,53 @@ - mode: NULLABLE name: activity_segment type: STRING +- mode: NULLABLE + name: newtab_default_ui + type: STRING +- name: newtab_weather_widget_enabled + type: BOOLEAN + mode: NULLABLE +- name: wallpaper_interactions + type: RECORD + mode: REPEATED + fields: + - name: wallpaper_selected_wallpaper + type: STRING + mode: NULLABLE + - name: wallpaper_clicks + type: INTEGER + mode: NULLABLE + - name: wallpaper_clicks_had_previous_wallpaper + type: INTEGER + mode: NULLABLE + - name: wallpaper_clicks_first_selected_wallpaper + type: INTEGER + mode: NULLABLE + - name: wallpaper_category_clicks + type: INTEGER + mode: NULLABLE + - name: wallpaper_highlight_dismissals + type: INTEGER + mode: NULLABLE + - name: wallpaper_highlight_cta_clicks + type: INTEGER + mode: NULLABLE +- name: weather_interactions + type: RECORD + mode: REPEATED + fields: + - name: weather_widget_impressions + type: INTEGER + mode: NULLABLE + - name: weather_widget_clicks + type: INTEGER + mode: NULLABLE + - name: weather_widget_load_errors + type: INTEGER + mode: NULLABLE + - name: weather_widget_change_display_to_detailed + type: INTEGER + mode: NULLABLE + - name: weather_widget_change_display_to_simple + type: INTEGER + mode: NULLABLE ```

Link to full diff

dataops-ci-bot commented 2 weeks ago

Integration report for "Merge branch 'main' into newtab_table_defaultUI_pocketID"

sql.diff

Click to expand! ```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql 2024-06-17 18:53:25.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql 2024-06-17 18:53:08.000000000 +0000 @@ -20,16 +20,6 @@ WHERE DATE(submission_timestamp) = @submission_date AND category IN ('newtab', 'topsites', 'newtab.search', 'newtab.search.ad', 'pocket') - AND name IN ( - 'closed', - 'opened', - 'impression', - 'issued', - 'click', - 'save', - 'topic_click', - 'dismiss' - ) ), visit_metadata AS ( SELECT @@ -54,6 +44,7 @@ ANY_VALUE(metrics.string.newtab_homepage_category) AS newtab_homepage_category, ANY_VALUE(metrics.string.newtab_newtab_category) AS newtab_newtab_category, ANY_VALUE(metrics.boolean.newtab_search_enabled) AS newtab_search_enabled, + ANY_VALUE(metrics.boolean.newtab_weather_enabled) AS newtab_weather_widget_enabled, ANY_VALUE(metrics.quantity.topsites_rows) AS topsites_rows, ANY_VALUE(metrics.string_list.newtab_blocked_sponsors) AS newtab_blocked_sponsors, ANY_VALUE(ping_info.experiments) AS experiments, @@ -110,6 +101,7 @@ events_unnested WHERE event_category IN ('newtab.search', 'newtab.search.ad') + AND event_name IN ('click', 'impression', 'issued') GROUP BY newtab_visit_id, search_engine, @@ -180,6 +172,7 @@ ) WHERE event_category = 'topsites' + AND event_name IN ('dismiss', 'click', 'impression') GROUP BY newtab_visit_id, topsite_tile_position, @@ -218,6 +211,8 @@ SELECT mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS pocket_story_position, + mozfun.map.get_key(event_details, "tile_id") AS pocket_tile_id, + mozfun.map.get_key(event_details, "recommendation_id") AS pocket_recommendation_id, COUNTIF(event_name = 'save') AS pocket_saves, COUNTIF(event_name = 'click') AS pocket_clicks, COUNTIF(event_name = 'impression') AS pocket_impressions, @@ -249,9 +244,12 @@ events_unnested WHERE event_category = 'pocket' + AND event_name IN ('impression', 'click', 'save') GROUP BY newtab_visit_id, - pocket_story_position + pocket_story_position, + pocket_tile_id, + pocket_recommendation_id ), pocket_summary AS ( SELECT @@ -259,6 +257,8 @@ ARRAY_AGG( STRUCT( pocket_story_position, + pocket_tile_id, + pocket_recommendation_id, pocket_impressions, sponsored_pocket_impressions, organic_pocket_impressions, @@ -275,6 +275,99 @@ GROUP BY newtab_visit_id ), +wallpaper_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + mozfun.map.get_key(event_details, "selected_wallpaper") AS wallpaper_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_click') AS wallpaper_clicks, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "true" + ) AS wallpaper_clicks_had_previous_wallpaper, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "false" + ) AS wallpaper_clicks_first_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_category_click') AS wallpaper_category_clicks, + COUNTIF(event_name = 'wallpaper_highlight_dismissed') AS wallpaper_highlight_dismissals, + COUNTIF(event_name = 'wallpaper_highlight_cta_click') AS wallpaper_highlight_cta_clicks + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'wallpaper_click', + 'wallpaper_category_click', + 'wallpaper_highlight_cta_clicks', + 'wallpaper_highlight_dismissed' + ) + GROUP BY + newtab_visit_id, + wallpaper_selected_wallpaper +), +wallpaper_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + wallpaper_selected_wallpaper, + wallpaper_clicks, + wallpaper_clicks_had_previous_wallpaper, + wallpaper_clicks_first_selected_wallpaper, + wallpaper_category_clicks, + wallpaper_highlight_dismissals, + wallpaper_highlight_cta_clicks + ) + ) AS wallpaper_interactions + FROM + wallpaper_events + GROUP BY + newtab_visit_id +), +weather_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + COUNTIF(event_name = 'weather_impression') AS weather_widget_impressions, + COUNTIF(event_name = 'weather_open_provider_url') AS weather_widget_clicks, + COUNTIF(event_name = 'weather_load_error') AS weather_widget_load_errors, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "detailed" + ) AS weather_widget_change_display_to_detailed, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "simple" + ) AS weather_widget_change_display_to_simple + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'weather_impression', + 'weather_open_provider_url', + 'weather_load_error', + 'weather_change_display' + ) + GROUP BY + newtab_visit_id +), +weather_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + weather_widget_impressions, + weather_widget_clicks, + weather_widget_load_errors, + weather_widget_change_display_to_detailed, + weather_widget_change_display_to_simple + ) + ) AS weather_interactions + FROM + weather_events + GROUP BY + newtab_visit_id +), combined_newtab_activity AS ( SELECT * @@ -289,6 +382,12 @@ LEFT JOIN pocket_summary USING (newtab_visit_id) + LEFT JOIN + wallpaper_summary + USING (newtab_visit_id) + LEFT JOIN + weather_summary + USING (newtab_visit_id) WHERE -- Keep only rows with interactions, unless we receive a valid newtab.opened event. -- This is meant to drop only interactions that only have a newtab.closed event on the same partition @@ -297,6 +396,8 @@ OR search_interactions IS NOT NULL OR topsite_tile_interactions IS NOT NULL OR pocket_interactions IS NOT NULL + OR wallpaper_interactions IS NOT NULL + OR weather_interactions IS NOT NULL ), client_profile_info AS ( SELECT @@ -311,7 +412,15 @@ client_id ) SELECT - * + *, + CASE + WHEN ( + (newtab_open_source = "about:home" AND newtab_homepage_category = "enabled") + OR (newtab_open_source = "about:newtab" AND newtab_newtab_category = "enabled") + ) + THEN "default" + ELSE "non-default" + END AS newtab_default_ui, FROM combined_newtab_activity LEFT JOIN diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml 2024-06-17 18:53:25.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml 2024-06-17 18:53:08.000000000 +0000 @@ -215,6 +215,12 @@ - mode: NULLABLE name: organic_pocket_saves type: INTEGER + - name: pocket_tile_id + type: STRING + mode: NULLABLE + - name: pocket_recommendation_id + type: STRING + mode: NULLABLE mode: REPEATED name: pocket_interactions type: RECORD @@ -224,3 +230,53 @@ - mode: NULLABLE name: activity_segment type: STRING +- mode: NULLABLE + name: newtab_default_ui + type: STRING +- name: newtab_weather_widget_enabled + type: BOOLEAN + mode: NULLABLE +- name: wallpaper_interactions + type: RECORD + mode: REPEATED + fields: + - name: wallpaper_selected_wallpaper + type: STRING + mode: NULLABLE + - name: wallpaper_clicks + type: INTEGER + mode: NULLABLE + - name: wallpaper_clicks_had_previous_wallpaper + type: INTEGER + mode: NULLABLE + - name: wallpaper_clicks_first_selected_wallpaper + type: INTEGER + mode: NULLABLE + - name: wallpaper_category_clicks + type: INTEGER + mode: NULLABLE + - name: wallpaper_highlight_dismissals + type: INTEGER + mode: NULLABLE + - name: wallpaper_highlight_cta_clicks + type: INTEGER + mode: NULLABLE +- name: weather_interactions + type: RECORD + mode: REPEATED + fields: + - name: weather_widget_impressions + type: INTEGER + mode: NULLABLE + - name: weather_widget_clicks + type: INTEGER + mode: NULLABLE + - name: weather_widget_load_errors + type: INTEGER + mode: NULLABLE + - name: weather_widget_change_display_to_detailed + type: INTEGER + mode: NULLABLE + - name: weather_widget_change_display_to_simple + type: INTEGER + mode: NULLABLE ```

Link to full diff

dataops-ci-bot commented 2 weeks ago

Integration report for "Merge branch 'main' into newtab_table_defaultUI_pocketID"

sql.diff

Click to expand! ```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql 2024-06-17 19:14:16.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/query.sql 2024-06-17 19:14:10.000000000 +0000 @@ -20,16 +20,6 @@ WHERE DATE(submission_timestamp) = @submission_date AND category IN ('newtab', 'topsites', 'newtab.search', 'newtab.search.ad', 'pocket') - AND name IN ( - 'closed', - 'opened', - 'impression', - 'issued', - 'click', - 'save', - 'topic_click', - 'dismiss' - ) ), visit_metadata AS ( SELECT @@ -54,6 +44,7 @@ ANY_VALUE(metrics.string.newtab_homepage_category) AS newtab_homepage_category, ANY_VALUE(metrics.string.newtab_newtab_category) AS newtab_newtab_category, ANY_VALUE(metrics.boolean.newtab_search_enabled) AS newtab_search_enabled, + ANY_VALUE(metrics.boolean.newtab_weather_enabled) AS newtab_weather_widget_enabled, ANY_VALUE(metrics.quantity.topsites_rows) AS topsites_rows, ANY_VALUE(metrics.string_list.newtab_blocked_sponsors) AS newtab_blocked_sponsors, ANY_VALUE(ping_info.experiments) AS experiments, @@ -110,6 +101,7 @@ events_unnested WHERE event_category IN ('newtab.search', 'newtab.search.ad') + AND event_name IN ('click', 'impression', 'issued') GROUP BY newtab_visit_id, search_engine, @@ -180,6 +172,7 @@ ) WHERE event_category = 'topsites' + AND event_name IN ('dismiss', 'click', 'impression') GROUP BY newtab_visit_id, topsite_tile_position, @@ -218,6 +211,8 @@ SELECT mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, SAFE_CAST(mozfun.map.get_key(event_details, "position") AS INT64) AS pocket_story_position, + mozfun.map.get_key(event_details, "tile_id") AS pocket_tile_id, + mozfun.map.get_key(event_details, "recommendation_id") AS pocket_recommendation_id, COUNTIF(event_name = 'save') AS pocket_saves, COUNTIF(event_name = 'click') AS pocket_clicks, COUNTIF(event_name = 'impression') AS pocket_impressions, @@ -249,9 +244,12 @@ events_unnested WHERE event_category = 'pocket' + AND event_name IN ('impression', 'click', 'save') GROUP BY newtab_visit_id, - pocket_story_position + pocket_story_position, + pocket_tile_id, + pocket_recommendation_id ), pocket_summary AS ( SELECT @@ -259,6 +257,8 @@ ARRAY_AGG( STRUCT( pocket_story_position, + pocket_tile_id, + pocket_recommendation_id, pocket_impressions, sponsored_pocket_impressions, organic_pocket_impressions, @@ -275,6 +275,99 @@ GROUP BY newtab_visit_id ), +wallpaper_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + mozfun.map.get_key(event_details, "selected_wallpaper") AS wallpaper_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_click') AS wallpaper_clicks, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "true" + ) AS wallpaper_clicks_had_previous_wallpaper, + COUNTIF( + event_name = 'wallpaper_click' + AND mozfun.map.get_key(event_details, "had_previous_wallpaper") = "false" + ) AS wallpaper_clicks_first_selected_wallpaper, + COUNTIF(event_name = 'wallpaper_category_click') AS wallpaper_category_clicks, + COUNTIF(event_name = 'wallpaper_highlight_dismissed') AS wallpaper_highlight_dismissals, + COUNTIF(event_name = 'wallpaper_highlight_cta_click') AS wallpaper_highlight_cta_clicks + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'wallpaper_click', + 'wallpaper_category_click', + 'wallpaper_highlight_cta_clicks', + 'wallpaper_highlight_dismissed' + ) + GROUP BY + newtab_visit_id, + wallpaper_selected_wallpaper +), +wallpaper_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + wallpaper_selected_wallpaper, + wallpaper_clicks, + wallpaper_clicks_had_previous_wallpaper, + wallpaper_clicks_first_selected_wallpaper, + wallpaper_category_clicks, + wallpaper_highlight_dismissals, + wallpaper_highlight_cta_clicks + ) + ) AS wallpaper_interactions + FROM + wallpaper_events + GROUP BY + newtab_visit_id +), +weather_events AS ( + SELECT + mozfun.map.get_key(event_details, "newtab_visit_id") AS newtab_visit_id, + COUNTIF(event_name = 'weather_impression') AS weather_widget_impressions, + COUNTIF(event_name = 'weather_open_provider_url') AS weather_widget_clicks, + COUNTIF(event_name = 'weather_load_error') AS weather_widget_load_errors, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "detailed" + ) AS weather_widget_change_display_to_detailed, + COUNTIF( + event_name = 'weather_change_display' + AND mozfun.map.get_key(event_details, "weather_display_mode") = "simple" + ) AS weather_widget_change_display_to_simple + FROM + events_unnested + WHERE + event_category = 'newtab' + AND event_name IN ( + 'weather_impression', + 'weather_open_provider_url', + 'weather_load_error', + 'weather_change_display' + ) + GROUP BY + newtab_visit_id +), +weather_summary AS ( + SELECT + newtab_visit_id, + ARRAY_AGG( + STRUCT( + weather_widget_impressions, + weather_widget_clicks, + weather_widget_load_errors, + weather_widget_change_display_to_detailed, + weather_widget_change_display_to_simple + ) + ) AS weather_interactions + FROM + weather_events + GROUP BY + newtab_visit_id +), combined_newtab_activity AS ( SELECT * @@ -289,6 +382,12 @@ LEFT JOIN pocket_summary USING (newtab_visit_id) + LEFT JOIN + wallpaper_summary + USING (newtab_visit_id) + LEFT JOIN + weather_summary + USING (newtab_visit_id) WHERE -- Keep only rows with interactions, unless we receive a valid newtab.opened event. -- This is meant to drop only interactions that only have a newtab.closed event on the same partition @@ -297,6 +396,8 @@ OR search_interactions IS NOT NULL OR topsite_tile_interactions IS NOT NULL OR pocket_interactions IS NOT NULL + OR wallpaper_interactions IS NOT NULL + OR weather_interactions IS NOT NULL ), client_profile_info AS ( SELECT @@ -311,7 +412,15 @@ client_id ) SELECT - * + *, + CASE + WHEN ( + (newtab_open_source = "about:home" AND newtab_homepage_category = "enabled") + OR (newtab_open_source = "about:newtab" AND newtab_newtab_category = "enabled") + ) + THEN "default" + ELSE "non-default" + END AS newtab_default_ui, FROM combined_newtab_activity LEFT JOIN diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml 2024-06-17 19:14:16.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry_derived/newtab_visits_v1/schema.yaml 2024-06-17 19:14:10.000000000 +0000 @@ -215,6 +215,12 @@ - mode: NULLABLE name: organic_pocket_saves type: INTEGER + - name: pocket_tile_id + type: STRING + mode: NULLABLE + - name: pocket_recommendation_id + type: STRING + mode: NULLABLE mode: REPEATED name: pocket_interactions type: RECORD @@ -224,3 +230,53 @@ - mode: NULLABLE name: activity_segment type: STRING +- mode: NULLABLE + name: newtab_default_ui + type: STRING +- name: newtab_weather_widget_enabled + type: BOOLEAN + mode: NULLABLE +- name: wallpaper_interactions + type: RECORD + mode: REPEATED + fields: + - name: wallpaper_selected_wallpaper + type: STRING + mode: NULLABLE + - name: wallpaper_clicks + type: INTEGER + mode: NULLABLE + - name: wallpaper_clicks_had_previous_wallpaper + type: INTEGER + mode: NULLABLE + - name: wallpaper_clicks_first_selected_wallpaper + type: INTEGER + mode: NULLABLE + - name: wallpaper_category_clicks + type: INTEGER + mode: NULLABLE + - name: wallpaper_highlight_dismissals + type: INTEGER + mode: NULLABLE + - name: wallpaper_highlight_cta_clicks + type: INTEGER + mode: NULLABLE +- name: weather_interactions + type: RECORD + mode: REPEATED + fields: + - name: weather_widget_impressions + type: INTEGER + mode: NULLABLE + - name: weather_widget_clicks + type: INTEGER + mode: NULLABLE + - name: weather_widget_load_errors + type: INTEGER + mode: NULLABLE + - name: weather_widget_change_display_to_detailed + type: INTEGER + mode: NULLABLE + - name: weather_widget_change_display_to_simple + type: INTEGER + mode: NULLABLE ```

Link to full diff