dataops-ci-bot
commented
2 months ago Integration report for "Reprocess past two days in jobs_by_organization_v1"
sql.diff
Click to expand!
```diff Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry: active_users Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry: active_users_mobile Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry: retention Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry: retention_clients Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry: retention_clients_mobile Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry: retention_mobile diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:42:58.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:31:52.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.interaction_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- name: submission_date +- mode: NULLABLE + name: submission_date type: DATE - mode: NULLABLE -- name: source +- mode: NULLABLE + name: source type: STRING - mode: NULLABLE -- name: event_type +- mode: NULLABLE + name: event_type type: STRING - mode: NULLABLE -- name: form_factor +- mode: NULLABLE + name: form_factor type: STRING - mode: NULLABLE -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: subdivision1 +- mode: NULLABLE + name: subdivision1 type: STRING - mode: NULLABLE -- name: advertiser +- mode: NULLABLE + name: advertiser type: STRING - mode: NULLABLE -- name: release_channel +- mode: NULLABLE + name: release_channel type: STRING - mode: NULLABLE -- name: position +- mode: NULLABLE + name: position type: INTEGER - mode: NULLABLE -- name: provider +- mode: NULLABLE + name: provider type: STRING - mode: NULLABLE -- name: match_type +- mode: NULLABLE + name: match_type type: STRING - mode: NULLABLE -- name: normalized_os +- mode: NULLABLE + name: normalized_os type: STRING - mode: NULLABLE -- name: suggest_data_sharing_enabled +- mode: NULLABLE + name: suggest_data_sharing_enabled type: BOOLEAN - mode: NULLABLE -- name: event_count +- mode: NULLABLE + name: event_count type: INTEGER - mode: NULLABLE -- name: user_count +- mode: NULLABLE + name: user_count type: INTEGER - mode: NULLABLE -- name: query_type +- mode: NULLABLE + name: query_type type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- name: submission_date +- mode: NULLABLE + name: submission_date type: DATE - mode: NULLABLE -- name: form_factor +- mode: NULLABLE + name: form_factor type: STRING - mode: NULLABLE -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: advertiser +- mode: NULLABLE + name: advertiser type: STRING - mode: NULLABLE -- name: normalized_os +- mode: NULLABLE + name: normalized_os type: STRING - mode: NULLABLE -- name: release_channel +- mode: NULLABLE + name: release_channel type: STRING - mode: NULLABLE -- name: position +- mode: NULLABLE + name: position type: INTEGER - mode: NULLABLE -- name: provider +- mode: NULLABLE + name: provider type: STRING - mode: NULLABLE -- name: match_type +- mode: NULLABLE + name: match_type type: STRING - mode: NULLABLE -- name: suggest_data_sharing_enabled +- mode: NULLABLE + name: suggest_data_sharing_enabled type: BOOLEAN - mode: NULLABLE -- name: impression_count +- mode: NULLABLE + name: impression_count type: INTEGER - mode: NULLABLE -- name: click_count +- mode: NULLABLE + name: click_count type: INTEGER - mode: NULLABLE -- name: query_type +- mode: NULLABLE + name: query_type type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -26,9 +26,6 @@ - name: adjust_network type: STRING mode: NULLABLE -- name: install_source - type: STRING - mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -48,10 +48,6 @@ description: 'The type of source of a client installation. ' -- name: install_source - type: STRING - mode: NULLABLE - description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:42:58.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:31:52.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:31:52.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: city +- mode: NULLABLE + name: city type: STRING - mode: NULLABLE -- name: datetime +- mode: NULLABLE + name: datetime type: TIMESTAMP - mode: NULLABLE -- name: proportion_undefined +- mode: NULLABLE + name: proportion_undefined type: FLOAT - mode: NULLABLE -- name: proportion_timeout +- mode: NULLABLE + name: proportion_timeout type: FLOAT - mode: NULLABLE -- name: proportion_abort +- mode: NULLABLE + name: proportion_abort type: FLOAT - mode: NULLABLE -- name: proportion_unreachable +- mode: NULLABLE + name: proportion_unreachable type: FLOAT - mode: NULLABLE -- name: proportion_terminated +- mode: NULLABLE + name: proportion_terminated type: FLOAT - mode: NULLABLE -- name: proportion_channel_open +- mode: NULLABLE + name: proportion_channel_open type: FLOAT - mode: NULLABLE -- name: avg_dns_success_time +- mode: NULLABLE + name: avg_dns_success_time type: FLOAT - mode: NULLABLE -- name: missing_dns_success +- mode: NULLABLE + name: missing_dns_success type: FLOAT - mode: NULLABLE -- name: avg_dns_failure_time +- mode: NULLABLE + name: avg_dns_failure_time type: FLOAT - mode: NULLABLE -- name: missing_dns_failure +- mode: NULLABLE + name: missing_dns_failure type: FLOAT - mode: NULLABLE -- name: count_dns_failure +- mode: NULLABLE + name: count_dns_failure type: FLOAT - mode: NULLABLE -- name: ssl_error_prop +- mode: NULLABLE + name: ssl_error_prop type: FLOAT - mode: NULLABLE -- name: avg_tls_handshake_time +- mode: NULLABLE + name: avg_tls_handshake_time type: FLOAT - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-05-23 19:42:58.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-05-23 19:31:52.000000000 +0000 @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -572,7 +572,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -582,7 +582,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -592,7 +592,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -663,7 +663,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -673,7 +673,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -683,7 +683,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -754,7 +754,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -764,7 +764,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -774,7 +774,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1555,7 +1555,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1565,7 +1565,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1575,7 +1575,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1646,7 +1646,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1656,7 +1656,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1666,7 +1666,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1737,7 +1737,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1747,7 +1747,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1757,7 +1757,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1828,7 +1828,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1838,7 +1838,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1848,7 +1848,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2000,7 +2000,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2010,7 +2010,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2081,7 +2081,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` + `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` UNION ALL SELECT submission_timestamp, @@ -2091,7 +2091,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` + `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2162,7 +2162,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -2172,7 +2172,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/jobs_by_organization_v1/query.py /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/jobs_by_organization_v1/query.py --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/jobs_by_organization_v1/query.py 2024-05-23 19:42:58.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/jobs_by_organization_v1/query.py 2024-05-23 19:31:05.000000000 +0000 @@ -3,6 +3,7 @@ """Put data from region-us.INFORMATION_SCHEMA.JOBS_BY_ORGANIZATION into a table.""" from argparse import ArgumentParser +from datetime import date, timedelta from google.cloud import bigquery @@ -14,14 +15,16 @@ ] parser = ArgumentParser(description=__doc__) -parser.add_argument("--date", required=True) # expect string with format yyyy-mm-dd +parser.add_argument( + "--date", required=True, type=date.fromisoformat +) # expect string with format yyyy-mm-dd parser.add_argument("--project", default="moz-fx-data-shared-prod") parser.add_argument("--destination_project", default="moz-fx-data-shared-prod") parser.add_argument("--destination_dataset", default="monitoring_derived") parser.add_argument("--destination_table", default="jobs_by_organization_v1") -def create_query(date, project): +def create_query(job_date: date, project: str): """Create query with filter for source projects.""" return f""" SELECT @@ -60,7 +63,7 @@ FROM `{project}.region-us.INFORMATION_SCHEMA.JOBS_BY_ORGANIZATION` WHERE - DATE(creation_time) = '{date}' + DATE(creation_time) = '{job_date.isoformat()}' AND ( ( project_id IN UNNEST({DEFAULT_PROJECTS}) @@ -83,18 +86,24 @@ args = parser.parse_args() project = args.project - partition = args.date.replace("-", "") - destination_table = f"{args.destination_project}.{args.destination_dataset}.{args.destination_table}${partition}" + # reprocess previous date to update jobs completed after the last run + for partition_date in (args.date - timedelta(days=1), args.date): + partition = partition_date.strftime("%Y%m%d") + destination_table = ( + f"{args.destination_project}.{args.destination_dataset}" + f".{args.destination_table}${partition}" + ) - # remove old partition in case of re-run client = bigquery.Client(project) - client.delete_table(destination_table, not_found_ok=True) - query = create_query(args.date, project) + query = create_query(partition_date, project) job_config = bigquery.QueryJobConfig( - destination=destination_table, write_disposition="WRITE_APPEND" + destination=destination_table, + write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE, ) - client.query(query, job_config=job_config).result() + result = client.query(query, job_config=job_config).result() + + print(f"Wrote {result.total_rows} rows to {destination_table}") if __name__ == "__main__": diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:31:52.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.main_v1` + `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -1,10 +1,7 @@ fields: -- name: build_hour - type: DATETIME - mode: NULLABLE -- name: geckoview_major_version - type: INTEGER - mode: NULLABLE -- name: n_pings - type: INTEGER - mode: NULLABLE +- type: DATETIME + name: build_hour +- type: INTEGER + name: geckoview_major_version +- type: INTEGER + name: n_pings diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:42:58.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:31:52.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:42:58.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:31:52.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:31:52.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:31:52.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:31:52.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:42:58.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-05-23 19:31:52.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/query.sql 2024-05-23 19:42:58.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/search_derived/mobile_search_clients_daily_v1/query.sql 2024-05-23 19:31:05.000000000 +0000 @@ -1,4 +1,6 @@ --- Query generated by bigquery-etl/search/mobile_search_clients_daily.py +-- Query generated by ./bqetl generate search +-- This file doesn't get overwritten by the generator. The generator output needs +-- to be written to this file manually. -- -- Older versions separate source and engine with an underscore instead of period -- Return array of form [source, engine] if key is valid, empty array otherwise @@ -496,7 +498,7 @@ metrics.counter.browser_total_uri_count, client_info.locale, FROM - org_mozilla_ios_klar.metrics AS org_mozilla_ios_klar_metrics + org_mozilla_ios_klar.metrics AS org_mozilla_klar_metrics ), fenix_baseline AS ( SELECT @@ -872,6 +874,7 @@ ANY_VALUE(sample_id) AS sample_id, udf.map_mode_last(ARRAY_CONCAT_AGG(experiments)) AS experiments, SUM(total_uri_count) AS total_uri_count, + CAST(NULL AS STRING) AS normalized_engine FROM combined_search_clients WHERE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -1,7 +1,9 @@ fields: - name: postal_district_code type: STRING - mode: NULLABLE + mode: REQUIRED + description: One-character Canadian postal district code. - name: province_code type: STRING mode: NULLABLE + description: Two-character Canadian province/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -1,28 +1,47 @@ fields: - name: name + description: Official country name per ISO 3166 type: STRING - mode: NULLABLE + mode: REQUIRED - name: code + description: ISO 3166 alpha-2 country code type: STRING - mode: NULLABLE + mode: REQUIRED - name: code_3 + description: ISO 3166 alpha-3 country code type: STRING - mode: NULLABLE + mode: REQUIRED - name: region_name + description: Region name. These are based on the UN Statistics Division standard + country or area codes for statistical use (M49), but with the "Americas" region + split into "North America" and "South America". type: STRING - mode: NULLABLE + mode: REQUIRED - name: subregion_name + description: Sub-region name. These are based on UN Statistics Division standard + country or area codes for statistical use (M49), but with the "Latin America and the + Caribbean" and "Sub-Saharan Africa" sub-regions split into more specific + sub-regions. type: STRING - mode: NULLABLE + mode: REQUIRED - name: pocket_available_on_newtab - type: BOOLEAN - mode: NULLABLE + description: Whether Pocket is available on the newtab page in this country. Note + that Pocket might only be available in certain locales/languages within a country. + type: BOOL + mode: REQUIRED - name: mozilla_vpn_available - type: BOOLEAN - mode: NULLABLE + description: Whether Mozilla VPN is available in this country. + type: BOOL + mode: REQUIRED - name: sponsored_tiles_available_on_newtab - type: BOOLEAN - mode: NULLABLE + description: Whether sponsored tiles are available on the newtab page in this country. + Note that Pocket might only be available in certain locales/languages within a + country. + type: BOOL + mode: REQUIRED - name: ads_value_tier + description: Lowercase label detailing the monetary value tier that Mozilla Ads + assign to that region based on market size and our existing products, e.g., tier + 1, tier 2, etc. type: STRING - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -1,7 +1,10 @@ fields: - name: name + description: An alias for a country's name (including misspellings and alternate + encodings). type: STRING - mode: NULLABLE + mode: REQUIRED - name: code + description: ISO 3166 alpha-2 country code type: STRING - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: incident +- mode: NULLABLE + name: incident type: STRING - mode: NULLABLE -- name: description +- mode: NULLABLE + name: description type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE -- name: product +- mode: NULLABLE + name: product type: STRING - mode: NULLABLE -- name: version +- mode: NULLABLE + name: version type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -1,16 +1,27 @@ fields: -- name: value +- mode: NULLABLE + description: Hex value assigned to the TLS cipher, in format like "0x00,0x84"; note + some values are ranges or contain wildcards + name: value type: STRING - mode: NULLABLE -- name: description +- mode: NULLABLE + description: Human-readable name of the TLS cipher + name: description type: STRING - mode: NULLABLE -- name: dtls_ok +- mode: NULLABLE + description: Any TLS cipher suite that is specified for use with DTLS MUST define + limits on the use of the associated AEAD function that preserves margins for both + confidentiality and integrity, as specified in [RFC-ietf-tls-dtls13-43] + name: dtls_ok type: BOOLEAN - mode: NULLABLE -- name: recommended +- mode: NULLABLE + description: Whether the TLS cipher is recommended by the IETF. If an item is not + marked as "recommended", it does not necessarily mean that it is flawed; rather, + it indicates that the item either has not been through the IETF consensus process, + has limited applicability, or is intended only for specific use cases + name: recommended type: BOOLEAN - mode: NULLABLE -- name: reference +- mode: NULLABLE + description: RFCs or associated reference material for the TLS cipher + name: reference type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -1,13 +1,17 @@ fields: - name: code_3 + description: ISO 639 alpha-3 language code. type: STRING - mode: NULLABLE + mode: REQUIRED - name: code_2 + description: ISO 639 alpha-2 language code (if any). type: STRING mode: NULLABLE - name: name + description: Language name. type: STRING - mode: NULLABLE + mode: REQUIRED - name: other_names + description: Other names for the language (if any). type: STRING mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -1,19 +1,19 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -1,25 +1,25 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: document_version +- mode: NULLABLE + name: document_version type: STRING - mode: NULLABLE -- name: path +- mode: NULLABLE + name: path type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: document_version +- mode: NULLABLE + name: document_version type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: path +- mode: NULLABLE + name: path type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -1,7 +1,7 @@ fields: - name: score type: NUMERIC - mode: NULLABLE + mode: REQUIRED - name: value type: NUMERIC - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-05-23 19:31:05.000000000 +0000 @@ -1,7 +1,9 @@ fields: - name: zip_code_prefix type: STRING - mode: NULLABLE + mode: REQUIRED + description: Three-digit US ZIP code prefix. - name: state_code type: STRING mode: NULLABLE + description: Two-character US state/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users/metadata.yaml 2024-05-23 19:31:52.000000000 +0000 @@ -0,0 +1,19 @@ +friendly_name: Active Users +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:dataops-managed/taar + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.fenix.active_users + - moz-fx-data-shared-prod.firefox_ios.active_users + - moz-fx-data-shared-prod.focus_android.active_users + - moz-fx-data-shared-prod.focus_ios.active_users + - moz-fx-data-shared-prod.klar_android.active_users + - moz-fx-data-shared-prod.klar_ios.active_users diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users/view.sql 2024-05-23 19:31:52.000000000 +0000 @@ -0,0 +1,33 @@ +-- Query generated via `kpi_support_metrics` SQL generator. +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.telemetry.active_users` +AS +SELECT + *, +FROM + `moz-fx-data-shared-prod.fenix.active_users` +UNION ALL +SELECT + *, +FROM + `moz-fx-data-shared-prod.focus_android.active_users` +UNION ALL +SELECT + *, +FROM + `moz-fx-data-shared-prod.firefox_ios.active_users` +UNION ALL +SELECT + *, +FROM + `moz-fx-data-shared-prod.focus_ios.active_users` +UNION ALL +SELECT + *, +FROM + `moz-fx-data-shared-prod.klar_ios.active_users` +UNION ALL +SELECT + *, +FROM + `moz-fx-data-shared-prod.klar_android.active_users` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users_mobile/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users_mobile/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users_mobile/metadata.yaml 2024-05-23 19:43:59.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users_mobile/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 @@ -1,19 +0,0 @@ -friendly_name: Active Users Mobile -description: |- - Please provide a description for the query -owners: [] -labels: {} -bigquery: null -workgroup_access: -- role: roles/bigquery.dataViewer - members: - - workgroup:dataops-managed/taar - - workgroup:mozilla-confidential -references: - view.sql: - - moz-fx-data-shared-prod.fenix.active_users - - moz-fx-data-shared-prod.firefox_ios.active_users - - moz-fx-data-shared-prod.focus_android.active_users - - moz-fx-data-shared-prod.focus_ios.active_users - - moz-fx-data-shared-prod.klar_android.active_users - - moz-fx-data-shared-prod.klar_ios.active_users diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users_mobile/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users_mobile/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users_mobile/view.sql 2024-05-23 19:42:57.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users_mobile/view.sql 1970-01-01 00:00:00.000000000 +0000 @@ -1,33 +0,0 @@ --- Query generated via `kpi_support_metrics` SQL generator. -CREATE OR REPLACE VIEW - `moz-fx-data-shared-prod.telemetry.active_users_mobile` -AS -SELECT - *, -FROM - `moz-fx-data-shared-prod.fenix.active_users` -UNION ALL -SELECT - *, -FROM - `moz-fx-data-shared-prod.focus_android.active_users` -UNION ALL -SELECT - *, -FROM - `moz-fx-data-shared-prod.firefox_ios.active_users` -UNION ALL -SELECT - *, -FROM - `moz-fx-data-shared-prod.focus_ios.active_users` -UNION ALL -SELECT - *, -FROM - `moz-fx-data-shared-prod.klar_ios.active_users` -UNION ALL -SELECT - *, -FROM - `moz-fx-data-shared-prod.klar_android.active_users` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention/metadata.yaml 2024-05-23 19:31:52.000000000 +0000 @@ -0,0 +1,15 @@ +friendly_name: Retention +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:dataops-managed/taar + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.fenix.retention + - moz-fx-data-shared-prod.firefox_ios.retention diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention/schema.yaml 2024-05-23 19:31:52.000000000 +0000 @@ -0,0 +1,76 @@ +fields: +- name: metric_date + type: DATE + mode: NULLABLE +- name: first_seen_date + type: DATE + mode: NULLABLE +- name: app_name + type: STRING + mode: NULLABLE +- name: normalized_channel + type: STRING + mode: NULLABLE +- name: country + type: STRING + mode: NULLABLE +- name: app_version + type: STRING + mode: NULLABLE +- name: locale + type: STRING + mode: NULLABLE +- name: adjust_ad_group + type: STRING + mode: NULLABLE +- name: adjust_campaign + type: STRING + mode: NULLABLE +- name: adjust_creative + type: STRING + mode: NULLABLE +- name: adjust_network + type: STRING + mode: NULLABLE +- name: ping_sent_metric_date + type: INTEGER + mode: NULLABLE +- name: ping_sent_week_4 + type: INTEGER + mode: NULLABLE +- name: active_metric_date + type: INTEGER + mode: NULLABLE +- name: retained_week_4 + type: INTEGER + mode: NULLABLE +- name: retained_week_4_new_profiles + type: INTEGER + mode: NULLABLE +- name: new_profiles_metric_date + type: INTEGER + mode: NULLABLE +- name: repeat_profiles + type: INTEGER + mode: NULLABLE +- name: lifecycle_stage + type: STRING + mode: NULLABLE +- name: play_store_attribution_campaign + type: STRING + mode: NULLABLE +- name: play_store_attribution_medium + type: STRING + mode: NULLABLE +- name: play_store_attribution_source + type: STRING + mode: NULLABLE +- name: meta_attribution_app + type: STRING + mode: NULLABLE +- name: install_source + type: STRING + mode: NULLABLE +- name: is_suspicious_device_client + type: BOOLEAN + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention/view.sql 2024-05-23 19:31:52.000000000 +0000 @@ -0,0 +1,31 @@ +-- Query generated via `kpi_support_metrics` SQL generator. +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.telemetry.retention` +AS +SELECT + * EXCEPT ( + play_store_attribution_campaign, + play_store_attribution_medium, + play_store_attribution_source, + meta_attribution_app, + install_source + ), + play_store_attribution_campaign, + play_store_attribution_medium, + play_store_attribution_source, + meta_attribution_app, + install_source, + CAST(NULL AS BOOLEAN) AS is_suspicious_device_client, +FROM + `moz-fx-data-shared-prod.fenix.retention` +UNION ALL +SELECT + * EXCEPT (is_suspicious_device_client), + CAST(NULL AS STRING) AS play_store_attribution_campaign, + CAST(NULL AS STRING) AS play_store_attribution_medium, + CAST(NULL AS STRING) AS play_store_attribution_source, + CAST(NULL AS STRING) AS meta_attribution_app, + CAST(NULL AS STRING) AS install_source, + is_suspicious_device_client, +FROM + `moz-fx-data-shared-prod.firefox_ios.retention` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention_clients/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention_clients/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention_clients/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention_clients/metadata.yaml 2024-05-23 19:31:52.000000000 +0000 @@ -0,0 +1,15 @@ +friendly_name: Retention Clients +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:dataops-managed/taar + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.fenix.retention_clients + - moz-fx-data-shared-prod.firefox_ios.retention_clients diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention_clients/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention_clients/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention_clients/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention_clients/view.sql 2024-05-23 19:31:52.000000000 +0000 @@ -0,0 +1,31 @@ +-- Query generated via `kpi_support_metrics` SQL generator. +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.telemetry.retention_clients` +AS +SELECT + * EXCEPT ( + play_store_attribution_campaign, + play_store_attribution_medium, + play_store_attribution_source, + meta_attribution_app, + install_source + ), + play_store_attribution_campaign, + play_store_attribution_medium, + play_store_attribution_source, + meta_attribution_app, + install_source, + CAST(NULL AS BOOLEAN) AS is_suspicious_device_client, +FROM + `moz-fx-data-shared-prod.fenix.retention_clients` +UNION ALL +SELECT + * EXCEPT (is_suspicious_device_client), + CAST(NULL AS STRING) AS play_store_attribution_campaign, + CAST(NULL AS STRING) AS play_store_attribution_medium, + CAST(NULL AS STRING) AS play_store_attribution_source, + CAST(NULL AS STRING) AS meta_attribution_app, + CAST(NULL AS STRING) AS install_source, + is_suspicious_device_client, +FROM + `moz-fx-data-shared-prod.firefox_ios.retention_clients` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention_clients_mobile/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention_clients_mobile/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/retention_clients_mobile/metadata. ```⚠️ Only part of the diff is displayed.
I noticed some jobs in the table are listed as running and don't have a
total_slot_msbecause they were still running at 2:00 UTC. This will query an additional previous day of jobs to update running jobs, which to me seems better than delaying the task by 4 hours or introducing an additional day delay.Checklist for reviewer:
<username>:<branch>of the fork as parameter. The parameter will also show up in the logs of themanual-trigger-required-for-forkCI task together with more detailed instructions.For modifications to schemas in restricted namespaces (see
CODEOWNERS):┆Issue is synchronized with this Jira Task