Deng 3996 add jinja config to snapshot

[chelseybeck]

Goal:

Default snapshot interval to 1 HOUR w/ ability to override the snapshot_interval value when triggering the dag w/ config

Checklist for reviewer:

• [ ] Commits should reference a bug or github issue, if relevant (if a bug is referenced, the pull request should include the bug number in the title).
• [ ] If the PR comes from a fork, trigger integration CI tests by running the Push to upstream workflow and provide the <username>:<branch> of the fork as parameter. The parameter will also show up in the logs of the manual-trigger-required-for-fork CI task together with more detailed instructions.
• [ ] If adding a new field to a query, ensure that the schema and dependent downstream schemas have been updated.
• [ ] When adding a new derived dataset, ensure that data is not available already (fully or partially) and recommend extending an existing dataset in favor of creating new ones. Data can be available in the bigquery-etl repository, looker-hub or in looker-spoke-default.

For modifications to schemas in restricted namespaces (see CODEOWNERS):

• [ ] Follow the change control procedure

┆Issue is synchronized with this Jira Task

[dataops-ci-bot]

Integration report for "adding templating for config override"

sql.diff

Click to expand!

```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_braze.py /tmp/workspace/generated-sql/dags/bqetl_braze.py --- /tmp/workspace/main-generated-sql/dags/bqetl_braze.py 2024-06-04 16:27:36.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_braze.py 2024-06-04 16:39:41.000000000 +0000 @@ -238,21 +238,10 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], date_partition_parameter=None, depends_on_past=False, + parameters=["{'snapshot_interval': '1 HOUR'}"], sql_file_path="sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql", ) - braze_external__users_previous_day_snapshot__v2 = bigquery_etl_query( - task_id="braze_external__users_previous_day_snapshot__v2", - destination_table=None, - dataset_id="braze_external", - project_id="moz-fx-data-shared-prod", - owner="cbeck@mozilla.com", - email=["cbeck@mozilla.com", "leli@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - sql_file_path="sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql", - ) - checks__fail_braze_derived__newsletters__v1 = bigquery_dq_check( task_id="checks__fail_braze_derived__newsletters__v1", source_table="newsletters_v1", @@ -480,7 +469,7 @@ ) braze_external__changed_users__v1.set_upstream( - braze_external__users_previous_day_snapshot__v2 + braze_external__users_previous_day_snapshot__v1 ) braze_external__changed_users__v1.set_upstream( diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_glean_usage.py /tmp/workspace/generated-sql/dags/bqetl_glean_usage.py --- /tmp/workspace/main-generated-sql/dags/bqetl_glean_usage.py 2024-06-04 16:27:36.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_glean_usage.py 2024-06-04 16:39:44.000000000 +0000 @@ -365,6 +365,20 @@ task_group=task_group_burnham, ) + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( + task_id="checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1", + source_table="baseline_clients_last_seen_v1", + dataset_id="accounts_cirrus_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=True, + owner="ascholtz@mozilla.com", + email=["ascholtz@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_accounts_cirrus, + ) + checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -1259,6 +1273,20 @@ task_group=task_group_pine, ) + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( + task_id="checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1", + source_table="baseline_clients_last_seen_v1", + dataset_id="accounts_cirrus_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=False, + owner="ascholtz@mozilla.com", + email=["ascholtz@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_accounts_cirrus, + ) + checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -5100,6 +5128,18 @@ burnham_derived__metrics_clients_daily__v1 ) + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + accounts_cirrus_derived__baseline_clients_last_seen__v1 + ) + + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 + ) + checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1.set_upstream( checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 ) @@ -5384,6 +5424,18 @@ pine_derived__baseline_clients_last_seen__v1 ) + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + accounts_cirrus_derived__baseline_clients_last_seen__v1 + ) + + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 + ) + checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1.set_upstream( checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 ) Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1: checks.sql Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external: users_previous_day_snapshot_v2 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/checks.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/checks.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/checks.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/checks.sql 2024-06-04 16:26:58.000000000 +0000 @@ -0,0 +1,88 @@ +-- Generated via bigquery_etl.glean_usage + +#warn +{{ is_unique(["client_id"], where="submission_date = @submission_date") }} + +#warn +{{ min_row_count(1, where="submission_date = @submission_date") }} + +#warn +{{ not_null([ + "submission_date", + "client_id", + "sample_id", + "first_seen_date", + "days_seen_bits", + "days_active_bits", + "days_created_profile_bits", + "days_seen_session_start_bits", + "days_seen_session_end_bits" + ], where="submission_date = @submission_date") }} + +#warn +SELECT + IF( + COUNTIF(normalized_channel NOT IN ("nightly", "aurora", "release", "Other", "beta", "esr")) > 0, + ERROR("Unexpected values for field normalized_channel detected."), + NULL + ) +FROM + `{{ project_id }}.{{ dataset_id }}.{{ table_name }}` +WHERE + submission_date = @submission_date; + +#warn +{{ matches_pattern(column="country", pattern="^[A-Z]{2}$", where="submission_date = @submission_date") }} + +#warn +{{ value_length(column="client_id", expected_length=36, where="submission_date = @submission_date") }} + +#fail +WITH daily AS ( + SELECT + submission_date, + COUNT(DISTINCT client_id) AS client_count + FROM + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_daily_v1` + WHERE + submission_date = @submission_date + AND sample_id IS NOT NULL + GROUP BY + submission_date +), +last_seen AS ( + SELECT + submission_date, + COUNT(DISTINCT client_id) AS client_count + FROM + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_last_seen_v1` + WHERE + submission_date = @submission_date + AND mozfun.bits28.days_since_seen(days_seen_bits) = 0 + GROUP BY + submission_date +), +check_results AS ( + SELECT + 1 - (last_seen.client_count / daily.client_count) AS difference_perc + FROM + daily + LEFT JOIN + last_seen + USING (submission_date) +) +SELECT + IF( + ABS((SELECT difference_perc FROM check_results)) > 0.001, + ERROR( + CONCAT( + "Results don't match by > 1%, baseline_clients_daily table has ", + STRING(((SELECT submission_date FROM daily))), + ": ", + ABS((SELECT client_count FROM daily)), + ". baseline_clients_last_seen has ", + IFNULL(((SELECT client_count FROM last_seen)), 0) + ) + ), + NULL + ); diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/metadata.yaml 2024-06-04 16:24:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/metadata.yaml 2024-06-04 16:36:56.000000000 +0000 @@ -40,6 +40,10 @@ members: - workgroup:mozilla-confidential references: + checks.sql: + - .. + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_daily_v1 + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_last_seen_v1 query.sql: - accounts_cirrus_derived.baseline_clients_daily_v1 - accounts_cirrus_derived.baseline_clients_last_seen_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:24:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:26:40.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.interaction_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml 2024-06-04 16:25:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml 2024-06-04 16:36:50.000000000 +0000 @@ -33,4 +33,4 @@ - .. query.sql: - moz-fx-data-shared-prod.braze_derived.users_v1 - - moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v2 + - moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql 2024-06-04 16:24:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql 2024-06-04 16:24:52.000000000 +0000 @@ -38,7 +38,7 @@ FROM `moz-fx-data-shared-prod.braze_derived.users_v1` current_users FULL OUTER JOIN - `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v2` previous_users --change back to v1 after run tomorrow + `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1` previous_users --change back to v1 after run tomorrow ON current_users.external_id = previous_users.external_id WHERE current_users.external_id IS NULL -- deleted rows diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml 2024-06-04 16:25:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml 2024-06-04 16:36:50.000000000 +0000 @@ -18,6 +18,8 @@ scheduling: dag_name: bqetl_braze date_partition_parameter: null + parameters: + - snapshot_interval: 1 HOUR bigquery: time_partitioning: null range_partitioning: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql 2024-06-04 16:24:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql 2024-06-04 16:24:52.000000000 +0000 @@ -2,7 +2,5 @@ CREATE SNAPSHOT TABLE `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1` CLONE `moz-fx-data-shared-prod.braze_derived.users_v1` FOR SYSTEM_TIME AS OF TIMESTAMP_SUB( CURRENT_TIMESTAMP(), - INTERVAL 12 HOUR + INTERVAL {{ snapshot_interval }} ); - - -- Chose 12 hours here because it's split between when the job last ran and now (yesterday). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/metadata.yaml 2024-06-04 16:25:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 @@ -1,34 +0,0 @@ -friendly_name: Braze User Profiles Previous Day Snapshot v2 -description: |- - This is a snapshot of the users table at the state it was in before - the most recent DAG run. Since the user profiles table updates once a day at - 10AM UTC, we can update this snapshot looking 12 hours back (yesterday). - - This snapshot allows us to compare changed values in order to generate the - JSON payloads for Braze's cloud data ingestion tool. - - See https://mozilla-hub.atlassian.net/browse/DENG-3182 -owners: -- cbeck@mozilla.com -labels: - schedule: daily - owner: cbeck - dag: bqetl_braze - owner1: cbeck -scheduling: - dag_name: bqetl_braze - date_partition_parameter: null -bigquery: - time_partitioning: null - range_partitioning: null - clustering: null -workgroup_access: -- role: roles/bigquery.dataViewer - members: - - workgroup:braze - - workgroup:braze/ingestion-mozilla-dev - - workgroup:braze/ingestion-firefox -- role: roles/bigquery.dataEditor - members: - - workgroup:braze/writers -references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql 2024-06-04 16:24:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql 1970-01-01 00:00:00.000000000 +0000 @@ -1,8 +0,0 @@ -DROP SNAPSHOT TABLE `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v2`; - -CREATE SNAPSHOT TABLE `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v2` CLONE `moz-fx-data-shared-prod.braze_derived.users_v1` FOR SYSTEM_TIME AS OF TIMESTAMP_SUB( - CURRENT_TIMESTAMP(), - INTERVAL 1 HOUR -); - - -- Chose 12 hours here because it's split between when the job last ran and now (yesterday). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-04 16:32:04.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: source + mode: NULLABLE +- name: source type: STRING -- mode: NULLABLE - name: event_type + mode: NULLABLE +- name: event_type type: STRING -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: subdivision1 + mode: NULLABLE +- name: subdivision1 type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: event_count + mode: NULLABLE +- name: event_count type: INTEGER -- mode: NULLABLE - name: user_count + mode: NULLABLE +- name: user_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-04 16:32:04.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: impression_count + mode: NULLABLE +- name: impression_count type: INTEGER -- mode: NULLABLE - name: click_count + mode: NULLABLE +- name: click_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-04 16:32:21.000000000 +0000 @@ -26,6 +26,9 @@ - name: adjust_network type: STRING mode: NULLABLE +- name: install_source + type: STRING + mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-04 16:32:19.000000000 +0000 @@ -48,6 +48,10 @@ description: 'The type of source of a client installation. ' +- name: install_source + type: STRING + mode: NULLABLE + description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:24:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:26:41.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:24:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:26:41.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -80,7 +80,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-04 16:33:10.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: country +- name: country type: STRING -- mode: NULLABLE - name: city + mode: NULLABLE +- name: city type: STRING -- mode: NULLABLE - name: datetime + mode: NULLABLE +- name: datetime type: TIMESTAMP -- mode: NULLABLE - name: proportion_undefined + mode: NULLABLE +- name: proportion_undefined type: FLOAT -- mode: NULLABLE - name: proportion_timeout + mode: NULLABLE +- name: proportion_timeout type: FLOAT -- mode: NULLABLE - name: proportion_abort + mode: NULLABLE +- name: proportion_abort type: FLOAT -- mode: NULLABLE - name: proportion_unreachable + mode: NULLABLE +- name: proportion_unreachable type: FLOAT -- mode: NULLABLE - name: proportion_terminated + mode: NULLABLE +- name: proportion_terminated type: FLOAT -- mode: NULLABLE - name: proportion_channel_open + mode: NULLABLE +- name: proportion_channel_open type: FLOAT -- mode: NULLABLE - name: avg_dns_success_time + mode: NULLABLE +- name: avg_dns_success_time type: FLOAT -- mode: NULLABLE - name: missing_dns_success + mode: NULLABLE +- name: missing_dns_success type: FLOAT -- mode: NULLABLE - name: avg_dns_failure_time + mode: NULLABLE +- name: avg_dns_failure_time type: FLOAT -- mode: NULLABLE - name: missing_dns_failure + mode: NULLABLE +- name: missing_dns_failure type: FLOAT -- mode: NULLABLE - name: count_dns_failure + mode: NULLABLE +- name: count_dns_failure type: FLOAT -- mode: NULLABLE - name: ssl_error_prop + mode: NULLABLE +- name: ssl_error_prop type: FLOAT -- mode: NULLABLE - name: avg_tls_handshake_time + mode: NULLABLE +- name: avg_tls_handshake_time type: FLOAT + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-04 16:24:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-04 16:28:17.000000000 +0000 @@ -45,7 +45,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -75,7 +75,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -572,7 +572,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -582,7 +582,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -592,7 +592,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -663,7 +663,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -673,7 +673,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -683,7 +683,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -754,7 +754,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -764,7 +764,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -774,7 +774,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1555,7 +1555,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1575,7 +1575,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1646,7 +1646,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1666,7 +1666,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1737,7 +1737,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1757,7 +1757,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1828,7 +1828,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1848,7 +1848,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1990,7 +1990,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2000,7 +2000,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2162,7 +2162,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -2172,7 +2172,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:24:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:26:41.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.main_v1` + `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-04 16:33:11.000000000 +0000 @@ -1,7 +1,10 @@ fields: -- type: DATETIME - name: build_hour -- type: INTEGER - name: geckoview_major_version -- type: INTEGER - name: n_pings +- name: build_hour + type: DATETIME + mode: NULLABLE +- name: geckoview_major_version + type: INTEGER + mode: NULLABLE +- name: n_pings + type: INTEGER + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:24:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:26:42.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:24:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:26:43.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:24:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:26:43.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:24:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:26:43.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:24:42.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:26:43.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:24:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:26:43.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-04 16:33:20.000000000 +0000 @@ -1,9 +1,7 @@ fields: - name: postal_district_code type: STRING - mode: REQUIRED - description: One-character Canadian postal district code. + mode: NULLABLE - name: province_code type: STRING mode: NULLABLE - description: Two-character Canadian province/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-04 16:33:19.000000000 +0000 @@ -1,47 +1,28 @@ fields: - name: name - description: Official country name per ISO 3166 type: STRING - mode: REQUIRED + mode: NULLABLE - name: code - description: ISO 3166 alpha-2 country code type: STRING - mode: REQUIRED + mode: NULLABLE - name: code_3 - description: ISO 3166 alpha-3 country code type: STRING - mode: REQUIRED + mode: NULLABLE - name: region_name - description: Region name. These are based on the UN Statistics Division standard - country or area codes for statistical use (M49), but with the "Americas" region - split into "North America" and "South America". type: STRING - mode: REQUIRED + mode: NULLABLE - name: subregion_name - description: Sub-region name. These are based on UN Statistics Division standard - country or area codes for statistical use (M49), but with the "Latin America and the - Caribbean" and "Sub-Saharan Africa" sub-regions split into more specific - sub-regions. type: STRING - mode: REQUIRED + mode: NULLABLE - name: pocket_available_on_newtab - description: Whether Pocket is available on the newtab page in this country. Note - that Pocket might only be available in certain locales/languages within a country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: mozilla_vpn_available - description: Whether Mozilla VPN is available in this country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: sponsored_tiles_available_on_newtab - description: Whether sponsored tiles are available on the newtab page in this country. - Note that Pocket might only be available in certain locales/languages within a - country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: ads_value_tier - description: Lowercase label detailing the monetary value tier that Mozilla Ads - assign to that region based on market size and our existing products, e.g., tier - 1, tier 2, etc. type: STRING - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-04 16:33:20.000000000 +0000 @@ -1,10 +1,7 @@ fields: - name: name - description: An alias for a country's name (including misspellings and alternate - encodings). type: STRING - mode: REQUIRED + mode: NULLABLE - name: code - description: ISO 3166 alpha-2 country code type: STRING - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-04 16:33:20.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: incident + mode: NULLABLE +- name: incident type: STRING -- mode: NULLABLE - name: description + mode: NULLABLE +- name: description type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING -- mode: NULLABLE - name: product + mode: NULLABLE +- name: product type: STRING -- mode: NULLABLE - name: version + mode: NULLABLE +- name: version type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-04 16:33:19.000000000 +0000 @@ -1,27 +1,16 @@ fields: -- mode: NULLABLE - description: Hex value assigned to the TLS cipher, in format like "0x00,0x84"; note - some values are ranges or contain wildcards - name: value +- name: value type: STRING -- mode: NULLABLE - description: Human-readable name of the TLS cipher - name: description + mode: NULLABLE +- name: description type: STRING -- mode: NULLABLE - description: Any TLS cipher suite that is specified for use with DTLS MUST define - limits on the use of the associated AEAD function that preserves margins for both - confidentiality and integrity, as specified in [RFC-ietf-tls-dtls13-43] - name: dtls_ok + mode: NULLABLE +- name: dtls_ok type: BOOLEAN -- mode: NULLABLE - description: Whether the TLS cipher is recommended by the IETF. If an item is not - marked as "recommended", it does not necessarily mean that it is flawed; rather, - it indicates that the item either has not been through the IETF consensus process, - has limited applicability, or is intended only for specific use cases - name: recommended + mode: NULLABLE +- name: recommended type: BOOLEAN -- mode: NULLABLE - description: RFCs or associated reference material for the TLS cipher - name: reference + mode: NULLABLE +- name: reference type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-04 16:33:19.000000000 +0000 @@ -1,17 +1,13 @@ fields: - name: code_3 - description: ISO 639 alpha-3 language code. type: STRING - mode: REQUIRED + mode: NULLABLE - name: code_2 - description: ISO 639 alpha-2 language code (if any). type: STRING mode: NULLABLE - name: name - description: Language name. type: STRING - mode: REQUIRED + mode: NULLABLE - name: other_names - description: Other names for the language (if any). type: STRING mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-04 16:33:19.000000000 +0000 @@ -1,19 +1,19 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-04 16:33:19.000000000 +0000 @@ -1,25 +1,25 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: document_version + mode: NULLABLE +- name: document_version type: STRING -- mode: NULLABLE - name: path + mode: NULLABLE +- name: path type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-04 16:33:19.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: document_version + mode: NULLABLE +- name: document_version type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-04 16:33:20.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: path + mode: NULLABLE +- name: path type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-04 16:33:20.000000000 +0000 @@ -1,7 +1,7 @@ fields: - name: score type: NUMERIC - mode: REQUIRED + mode: NULLABLE - name: value type: NUMERIC - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-04 16:24:05.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-04 16:33:19.000000000 +0000 @@ -1,9 +1,7 @@ fields: - name: zip_code_prefix type: STRING - mode: REQUIRED - description: Three-digit US ZIP code prefix. + mode: NULLABLE - name: state_code type: STRING mode: NULLABLE - description: Two-character US state/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml 2024-06-04 16:24:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml 2024-06-04 16:34:05.000000000 +0000 @@ -2,21 +2,18 @@ - name: date type: DATE mode: NULLABLE - description: null - name: product type: STRING mode: NULLABLE - name: category type: STRING mode: NULLABLE - description: null - name: channel type: STRING mode: NULLABLE - name: build_number type: INTEGER mode: NULLABLE - description: null - name: release_date type: DATE mode: NULLABLE ```

Link to full diff

[dataops-ci-bot]

Integration report for "Merge branch 'main' into DENG-3996-add-jinja-config"

sql.diff

Click to expand!

```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_braze.py /tmp/workspace/generated-sql/dags/bqetl_braze.py --- /tmp/workspace/main-generated-sql/dags/bqetl_braze.py 2024-06-04 16:43:26.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_braze.py 2024-06-04 16:55:23.000000000 +0000 @@ -238,21 +238,10 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], date_partition_parameter=None, depends_on_past=False, + parameters=["snapshot_interval:INT64:1"], sql_file_path="sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql", ) - braze_external__users_previous_day_snapshot__v2 = bigquery_etl_query( - task_id="braze_external__users_previous_day_snapshot__v2", - destination_table=None, - dataset_id="braze_external", - project_id="moz-fx-data-shared-prod", - owner="cbeck@mozilla.com", - email=["cbeck@mozilla.com", "leli@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - sql_file_path="sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql", - ) - checks__fail_braze_derived__newsletters__v1 = bigquery_dq_check( task_id="checks__fail_braze_derived__newsletters__v1", source_table="newsletters_v1", @@ -480,7 +469,7 @@ ) braze_external__changed_users__v1.set_upstream( - braze_external__users_previous_day_snapshot__v2 + braze_external__users_previous_day_snapshot__v1 ) braze_external__changed_users__v1.set_upstream( Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external: users_previous_day_snapshot_v2 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:05.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.interaction_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml 2024-06-04 16:41:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml 2024-06-04 16:52:26.000000000 +0000 @@ -33,4 +33,4 @@ - .. query.sql: - moz-fx-data-shared-prod.braze_derived.users_v1 - - moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v2 + - moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql 2024-06-04 16:39:49.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql 2024-06-04 16:40:24.000000000 +0000 @@ -38,7 +38,7 @@ FROM `moz-fx-data-shared-prod.braze_derived.users_v1` current_users FULL OUTER JOIN - `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v2` previous_users --change back to v1 after run tomorrow + `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1` previous_users --change back to v1 after run tomorrow ON current_users.external_id = previous_users.external_id WHERE current_users.external_id IS NULL -- deleted rows diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml 2024-06-04 16:41:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml 2024-06-04 16:52:26.000000000 +0000 @@ -18,6 +18,8 @@ scheduling: dag_name: bqetl_braze date_partition_parameter: null + parameters: + - snapshot_interval:INT64:1 bigquery: time_partitioning: null range_partitioning: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql 2024-06-04 16:39:49.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql 2024-06-04 16:40:24.000000000 +0000 @@ -2,7 +2,5 @@ CREATE SNAPSHOT TABLE `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1` CLONE `moz-fx-data-shared-prod.braze_derived.users_v1` FOR SYSTEM_TIME AS OF TIMESTAMP_SUB( CURRENT_TIMESTAMP(), - INTERVAL 12 HOUR + INTERVAL snapshot_interval HOUR ); - - -- Chose 12 hours here because it's split between when the job last ran and now (yesterday). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/metadata.yaml 2024-06-04 16:41:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 @@ -1,34 +0,0 @@ -friendly_name: Braze User Profiles Previous Day Snapshot v2 -description: |- - This is a snapshot of the users table at the state it was in before - the most recent DAG run. Since the user profiles table updates once a day at - 10AM UTC, we can update this snapshot looking 12 hours back (yesterday). - - This snapshot allows us to compare changed values in order to generate the - JSON payloads for Braze's cloud data ingestion tool. - - See https://mozilla-hub.atlassian.net/browse/DENG-3182 -owners: -- cbeck@mozilla.com -labels: - schedule: daily - owner: cbeck - dag: bqetl_braze - owner1: cbeck -scheduling: - dag_name: bqetl_braze - date_partition_parameter: null -bigquery: - time_partitioning: null - range_partitioning: null - clustering: null -workgroup_access: -- role: roles/bigquery.dataViewer - members: - - workgroup:braze - - workgroup:braze/ingestion-mozilla-dev - - workgroup:braze/ingestion-firefox -- role: roles/bigquery.dataEditor - members: - - workgroup:braze/writers -references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql 2024-06-04 16:39:49.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql 1970-01-01 00:00:00.000000000 +0000 @@ -1,8 +0,0 @@ -DROP SNAPSHOT TABLE `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v2`; - -CREATE SNAPSHOT TABLE `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v2` CLONE `moz-fx-data-shared-prod.braze_derived.users_v1` FOR SYSTEM_TIME AS OF TIMESTAMP_SUB( - CURRENT_TIMESTAMP(), - INTERVAL 1 HOUR -); - - -- Chose 12 hours here because it's split between when the job last ran and now (yesterday). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-04 16:48:20.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: source + mode: NULLABLE +- name: source type: STRING -- mode: NULLABLE - name: event_type + mode: NULLABLE +- name: event_type type: STRING -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: subdivision1 + mode: NULLABLE +- name: subdivision1 type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: event_count + mode: NULLABLE +- name: event_count type: INTEGER -- mode: NULLABLE - name: user_count + mode: NULLABLE +- name: user_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-04 16:48:20.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: impression_count + mode: NULLABLE +- name: impression_count type: INTEGER -- mode: NULLABLE - name: click_count + mode: NULLABLE +- name: click_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-04 16:47:35.000000000 +0000 @@ -26,6 +26,9 @@ - name: adjust_network type: STRING mode: NULLABLE +- name: install_source + type: STRING + mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-04 16:47:35.000000000 +0000 @@ -48,6 +48,10 @@ description: 'The type of source of a client installation. ' +- name: install_source + type: STRING + mode: NULLABLE + description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:05.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:25.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:06.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -80,7 +80,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-04 16:48:14.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: country +- name: country type: STRING -- mode: NULLABLE - name: city + mode: NULLABLE +- name: city type: STRING -- mode: NULLABLE - name: datetime + mode: NULLABLE +- name: datetime type: TIMESTAMP -- mode: NULLABLE - name: proportion_undefined + mode: NULLABLE +- name: proportion_undefined type: FLOAT -- mode: NULLABLE - name: proportion_timeout + mode: NULLABLE +- name: proportion_timeout type: FLOAT -- mode: NULLABLE - name: proportion_abort + mode: NULLABLE +- name: proportion_abort type: FLOAT -- mode: NULLABLE - name: proportion_unreachable + mode: NULLABLE +- name: proportion_unreachable type: FLOAT -- mode: NULLABLE - name: proportion_terminated + mode: NULLABLE +- name: proportion_terminated type: FLOAT -- mode: NULLABLE - name: proportion_channel_open + mode: NULLABLE +- name: proportion_channel_open type: FLOAT -- mode: NULLABLE - name: avg_dns_success_time + mode: NULLABLE +- name: avg_dns_success_time type: FLOAT -- mode: NULLABLE - name: missing_dns_success + mode: NULLABLE +- name: missing_dns_success type: FLOAT -- mode: NULLABLE - name: avg_dns_failure_time + mode: NULLABLE +- name: avg_dns_failure_time type: FLOAT -- mode: NULLABLE - name: missing_dns_failure + mode: NULLABLE +- name: missing_dns_failure type: FLOAT -- mode: NULLABLE - name: count_dns_failure + mode: NULLABLE +- name: count_dns_failure type: FLOAT -- mode: NULLABLE - name: ssl_error_prop + mode: NULLABLE +- name: ssl_error_prop type: FLOAT -- mode: NULLABLE - name: avg_tls_handshake_time + mode: NULLABLE +- name: avg_tls_handshake_time type: FLOAT + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-04 16:40:25.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-04 16:43:39.000000000 +0000 @@ -45,7 +45,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -75,7 +75,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -572,7 +572,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -592,7 +592,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -663,7 +663,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -683,7 +683,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -754,7 +754,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -774,7 +774,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1555,7 +1555,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1565,7 +1565,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1575,7 +1575,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1646,7 +1646,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1656,7 +1656,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1666,7 +1666,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1737,7 +1737,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1747,7 +1747,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1757,7 +1757,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1828,7 +1828,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1838,7 +1838,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1848,7 +1848,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1990,7 +1990,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2000,7 +2000,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2081,7 +2081,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` + `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` UNION ALL SELECT submission_timestamp, @@ -2091,7 +2091,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` + `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2162,7 +2162,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -2172,7 +2172,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:25.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:05.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.main_v1` + `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-04 16:47:24.000000000 +0000 @@ -1,7 +1,10 @@ fields: -- type: DATETIME - name: build_hour -- type: INTEGER - name: geckoview_major_version -- type: INTEGER - name: n_pings +- name: build_hour + type: DATETIME + mode: NULLABLE +- name: geckoview_major_version + type: INTEGER + mode: NULLABLE +- name: n_pings + type: INTEGER + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:06.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:07.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:07.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:25.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:07.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:08.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:08.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-04 16:49:36.000000000 +0000 @@ -1,9 +1,7 @@ fields: - name: postal_district_code type: STRING - mode: REQUIRED - description: One-character Canadian postal district code. + mode: NULLABLE - name: province_code type: STRING mode: NULLABLE - description: Two-character Canadian province/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-04 16:49:37.000000000 +0000 @@ -1,47 +1,28 @@ fields: - name: name - description: Official country name per ISO 3166 type: STRING - mode: REQUIRED + mode: NULLABLE - name: code - description: ISO 3166 alpha-2 country code type: STRING - mode: REQUIRED + mode: NULLABLE - name: code_3 - description: ISO 3166 alpha-3 country code type: STRING - mode: REQUIRED + mode: NULLABLE - name: region_name - description: Region name. These are based on the UN Statistics Division standard - country or area codes for statistical use (M49), but with the "Americas" region - split into "North America" and "South America". type: STRING - mode: REQUIRED + mode: NULLABLE - name: subregion_name - description: Sub-region name. These are based on UN Statistics Division standard - country or area codes for statistical use (M49), but with the "Latin America and the - Caribbean" and "Sub-Saharan Africa" sub-regions split into more specific - sub-regions. type: STRING - mode: REQUIRED + mode: NULLABLE - name: pocket_available_on_newtab - description: Whether Pocket is available on the newtab page in this country. Note - that Pocket might only be available in certain locales/languages within a country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: mozilla_vpn_available - description: Whether Mozilla VPN is available in this country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: sponsored_tiles_available_on_newtab - description: Whether sponsored tiles are available on the newtab page in this country. - Note that Pocket might only be available in certain locales/languages within a - country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: ads_value_tier - description: Lowercase label detailing the monetary value tier that Mozilla Ads - assign to that region based on market size and our existing products, e.g., tier - 1, tier 2, etc. type: STRING - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-04 16:49:36.000000000 +0000 @@ -1,10 +1,7 @@ fields: - name: name - description: An alias for a country's name (including misspellings and alternate - encodings). type: STRING - mode: REQUIRED + mode: NULLABLE - name: code - description: ISO 3166 alpha-2 country code type: STRING - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-04 16:49:37.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: incident + mode: NULLABLE +- name: incident type: STRING -- mode: NULLABLE - name: description + mode: NULLABLE +- name: description type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING -- mode: NULLABLE - name: product + mode: NULLABLE +- name: product type: STRING -- mode: NULLABLE - name: version + mode: NULLABLE +- name: version type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-04 16:49:37.000000000 +0000 @@ -1,27 +1,16 @@ fields: -- mode: NULLABLE - description: Hex value assigned to the TLS cipher, in format like "0x00,0x84"; note - some values are ranges or contain wildcards - name: value +- name: value type: STRING -- mode: NULLABLE - description: Human-readable name of the TLS cipher - name: description + mode: NULLABLE +- name: description type: STRING -- mode: NULLABLE - description: Any TLS cipher suite that is specified for use with DTLS MUST define - limits on the use of the associated AEAD function that preserves margins for both - confidentiality and integrity, as specified in [RFC-ietf-tls-dtls13-43] - name: dtls_ok + mode: NULLABLE +- name: dtls_ok type: BOOLEAN -- mode: NULLABLE - description: Whether the TLS cipher is recommended by the IETF. If an item is not - marked as "recommended", it does not necessarily mean that it is flawed; rather, - it indicates that the item either has not been through the IETF consensus process, - has limited applicability, or is intended only for specific use cases - name: recommended + mode: NULLABLE +- name: recommended type: BOOLEAN -- mode: NULLABLE - description: RFCs or associated reference material for the TLS cipher - name: reference + mode: NULLABLE +- name: reference type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-04 16:49:36.000000000 +0000 @@ -1,17 +1,13 @@ fields: - name: code_3 - description: ISO 639 alpha-3 language code. type: STRING - mode: REQUIRED + mode: NULLABLE - name: code_2 - description: ISO 639 alpha-2 language code (if any). type: STRING mode: NULLABLE - name: name - description: Language name. type: STRING - mode: REQUIRED + mode: NULLABLE - name: other_names - description: Other names for the language (if any). type: STRING mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-04 16:49:36.000000000 +0000 @@ -1,19 +1,19 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-04 16:49:36.000000000 +0000 @@ -1,25 +1,25 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: document_version + mode: NULLABLE +- name: document_version type: STRING -- mode: NULLABLE - name: path + mode: NULLABLE +- name: path type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-04 16:49:36.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: document_version + mode: NULLABLE +- name: document_version type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-04 16:49:37.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: path + mode: NULLABLE +- name: path type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-04 16:49:37.000000000 +0000 @@ -1,7 +1,7 @@ fields: - name: score type: NUMERIC - mode: REQUIRED + mode: NULLABLE - name: value type: NUMERIC - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-04 16:39:48.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-04 16:49:36.000000000 +0000 @@ -1,9 +1,7 @@ fields: - name: zip_code_prefix type: STRING - mode: REQUIRED - description: Three-digit US ZIP code prefix. + mode: NULLABLE - name: state_code type: STRING mode: NULLABLE - description: Two-character US state/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:05.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_live.video_index_v1` + `moz-fx-data-shared-prod.viu_politica_live.main_events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_live.main_events_v1` + `moz-fx-data-shared-prod.viu_politica_live.video_index_v1` ) CROSS JOIN UNNEST(events) AS event, ```

Link to full diff

[dataops-ci-bot]

Integration report for "Update sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml"

sql.diff

Click to expand!

```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_braze.py /tmp/workspace/generated-sql/dags/bqetl_braze.py --- /tmp/workspace/main-generated-sql/dags/bqetl_braze.py 2024-06-04 16:43:13.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_braze.py 2024-06-04 16:55:28.000000000 +0000 @@ -238,21 +238,10 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], date_partition_parameter=None, depends_on_past=False, + parameters=["snapshot_interval:INT64:1"], sql_file_path="sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql", ) - braze_external__users_previous_day_snapshot__v2 = bigquery_etl_query( - task_id="braze_external__users_previous_day_snapshot__v2", - destination_table=None, - dataset_id="braze_external", - project_id="moz-fx-data-shared-prod", - owner="cbeck@mozilla.com", - email=["cbeck@mozilla.com", "leli@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - sql_file_path="sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql", - ) - checks__fail_braze_derived__newsletters__v1 = bigquery_dq_check( task_id="checks__fail_braze_derived__newsletters__v1", source_table="newsletters_v1", @@ -480,7 +469,7 @@ ) braze_external__changed_users__v1.set_upstream( - braze_external__users_previous_day_snapshot__v2 + braze_external__users_previous_day_snapshot__v1 ) braze_external__changed_users__v1.set_upstream( diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_glean_usage.py /tmp/workspace/generated-sql/dags/bqetl_glean_usage.py --- /tmp/workspace/main-generated-sql/dags/bqetl_glean_usage.py 2024-06-04 16:43:13.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_glean_usage.py 2024-06-04 16:55:31.000000000 +0000 @@ -365,6 +365,20 @@ task_group=task_group_burnham, ) + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( + task_id="checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1", + source_table="baseline_clients_last_seen_v1", + dataset_id="accounts_cirrus_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=True, + owner="ascholtz@mozilla.com", + email=["ascholtz@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_accounts_cirrus, + ) + checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -1259,6 +1273,20 @@ task_group=task_group_pine, ) + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( + task_id="checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1", + source_table="baseline_clients_last_seen_v1", + dataset_id="accounts_cirrus_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=False, + owner="ascholtz@mozilla.com", + email=["ascholtz@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_accounts_cirrus, + ) + checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -5100,6 +5128,18 @@ burnham_derived__metrics_clients_daily__v1 ) + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + accounts_cirrus_derived__baseline_clients_last_seen__v1 + ) + + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 + ) + checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1.set_upstream( checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 ) @@ -5384,6 +5424,18 @@ pine_derived__baseline_clients_last_seen__v1 ) + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + accounts_cirrus_derived__baseline_clients_last_seen__v1 + ) + + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 + ) + checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1.set_upstream( checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 ) Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1: checks.sql Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external: users_previous_day_snapshot_v2 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/checks.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/checks.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/checks.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/checks.sql 2024-06-04 16:42:19.000000000 +0000 @@ -0,0 +1,88 @@ +-- Generated via bigquery_etl.glean_usage + +#warn +{{ is_unique(["client_id"], where="submission_date = @submission_date") }} + +#warn +{{ min_row_count(1, where="submission_date = @submission_date") }} + +#warn +{{ not_null([ + "submission_date", + "client_id", + "sample_id", + "first_seen_date", + "days_seen_bits", + "days_active_bits", + "days_created_profile_bits", + "days_seen_session_start_bits", + "days_seen_session_end_bits" + ], where="submission_date = @submission_date") }} + +#warn +SELECT + IF( + COUNTIF(normalized_channel NOT IN ("nightly", "aurora", "release", "Other", "beta", "esr")) > 0, + ERROR("Unexpected values for field normalized_channel detected."), + NULL + ) +FROM + `{{ project_id }}.{{ dataset_id }}.{{ table_name }}` +WHERE + submission_date = @submission_date; + +#warn +{{ matches_pattern(column="country", pattern="^[A-Z]{2}$", where="submission_date = @submission_date") }} + +#warn +{{ value_length(column="client_id", expected_length=36, where="submission_date = @submission_date") }} + +#fail +WITH daily AS ( + SELECT + submission_date, + COUNT(DISTINCT client_id) AS client_count + FROM + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_daily_v1` + WHERE + submission_date = @submission_date + AND sample_id IS NOT NULL + GROUP BY + submission_date +), +last_seen AS ( + SELECT + submission_date, + COUNT(DISTINCT client_id) AS client_count + FROM + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_last_seen_v1` + WHERE + submission_date = @submission_date + AND mozfun.bits28.days_since_seen(days_seen_bits) = 0 + GROUP BY + submission_date +), +check_results AS ( + SELECT + 1 - (last_seen.client_count / daily.client_count) AS difference_perc + FROM + daily + LEFT JOIN + last_seen + USING (submission_date) +) +SELECT + IF( + ABS((SELECT difference_perc FROM check_results)) > 0.001, + ERROR( + CONCAT( + "Results don't match by > 1%, baseline_clients_daily table has ", + STRING(((SELECT submission_date FROM daily))), + ": ", + ABS((SELECT client_count FROM daily)), + ". baseline_clients_last_seen has ", + IFNULL(((SELECT client_count FROM last_seen)), 0) + ) + ), + NULL + ); diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/metadata.yaml 2024-06-04 16:40:15.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/metadata.yaml 2024-06-04 16:52:33.000000000 +0000 @@ -40,6 +40,10 @@ members: - workgroup:mozilla-confidential references: + checks.sql: + - .. + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_daily_v1 + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_last_seen_v1 query.sql: - accounts_cirrus_derived.baseline_clients_daily_v1 - accounts_cirrus_derived.baseline_clients_last_seen_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:14.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:02.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.events_v1` + `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml 2024-06-04 16:41:17.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml 2024-06-04 16:52:30.000000000 +0000 @@ -33,4 +33,4 @@ - .. query.sql: - moz-fx-data-shared-prod.braze_derived.users_v1 - - moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v2 + - moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql 2024-06-04 16:39:42.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql 2024-06-04 16:40:12.000000000 +0000 @@ -38,7 +38,7 @@ FROM `moz-fx-data-shared-prod.braze_derived.users_v1` current_users FULL OUTER JOIN - `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v2` previous_users --change back to v1 after run tomorrow + `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1` previous_users --change back to v1 after run tomorrow ON current_users.external_id = previous_users.external_id WHERE current_users.external_id IS NULL -- deleted rows diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml 2024-06-04 16:41:17.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml 2024-06-04 16:52:30.000000000 +0000 @@ -18,6 +18,8 @@ scheduling: dag_name: bqetl_braze date_partition_parameter: null + parameters: + - snapshot_interval:INT64:1 bigquery: time_partitioning: null range_partitioning: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql 2024-06-04 16:39:42.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql 2024-06-04 16:40:12.000000000 +0000 @@ -2,7 +2,5 @@ CREATE SNAPSHOT TABLE `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1` CLONE `moz-fx-data-shared-prod.braze_derived.users_v1` FOR SYSTEM_TIME AS OF TIMESTAMP_SUB( CURRENT_TIMESTAMP(), - INTERVAL 12 HOUR + INTERVAL snapshot_interval HOUR ); - - -- Chose 12 hours here because it's split between when the job last ran and now (yesterday). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/metadata.yaml 2024-06-04 16:41:17.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 @@ -1,34 +0,0 @@ -friendly_name: Braze User Profiles Previous Day Snapshot v2 -description: |- - This is a snapshot of the users table at the state it was in before - the most recent DAG run. Since the user profiles table updates once a day at - 10AM UTC, we can update this snapshot looking 12 hours back (yesterday). - - This snapshot allows us to compare changed values in order to generate the - JSON payloads for Braze's cloud data ingestion tool. - - See https://mozilla-hub.atlassian.net/browse/DENG-3182 -owners: -- cbeck@mozilla.com -labels: - schedule: daily - owner: cbeck - dag: bqetl_braze - owner1: cbeck -scheduling: - dag_name: bqetl_braze - date_partition_parameter: null -bigquery: - time_partitioning: null - range_partitioning: null - clustering: null -workgroup_access: -- role: roles/bigquery.dataViewer - members: - - workgroup:braze - - workgroup:braze/ingestion-mozilla-dev - - workgroup:braze/ingestion-firefox -- role: roles/bigquery.dataEditor - members: - - workgroup:braze/writers -references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql 2024-06-04 16:39:42.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql 1970-01-01 00:00:00.000000000 +0000 @@ -1,8 +0,0 @@ -DROP SNAPSHOT TABLE `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v2`; - -CREATE SNAPSHOT TABLE `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v2` CLONE `moz-fx-data-shared-prod.braze_derived.users_v1` FOR SYSTEM_TIME AS OF TIMESTAMP_SUB( - CURRENT_TIMESTAMP(), - INTERVAL 1 HOUR -); - - -- Chose 12 hours here because it's split between when the job last ran and now (yesterday). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-04 16:48:17.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: source + mode: NULLABLE +- name: source type: STRING -- mode: NULLABLE - name: event_type + mode: NULLABLE +- name: event_type type: STRING -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: subdivision1 + mode: NULLABLE +- name: subdivision1 type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: event_count + mode: NULLABLE +- name: event_count type: INTEGER -- mode: NULLABLE - name: user_count + mode: NULLABLE +- name: user_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-04 16:48:17.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: impression_count + mode: NULLABLE +- name: impression_count type: INTEGER -- mode: NULLABLE - name: click_count + mode: NULLABLE +- name: click_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-04 16:47:39.000000000 +0000 @@ -26,6 +26,9 @@ - name: adjust_network type: STRING mode: NULLABLE +- name: install_source + type: STRING + mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-04 16:47:39.000000000 +0000 @@ -48,6 +48,10 @@ description: 'The type of source of a client installation. ' +- name: install_source + type: STRING + mode: NULLABLE + description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:15.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:02.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-04 16:48:12.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: country +- name: country type: STRING -- mode: NULLABLE - name: city + mode: NULLABLE +- name: city type: STRING -- mode: NULLABLE - name: datetime + mode: NULLABLE +- name: datetime type: TIMESTAMP -- mode: NULLABLE - name: proportion_undefined + mode: NULLABLE +- name: proportion_undefined type: FLOAT -- mode: NULLABLE - name: proportion_timeout + mode: NULLABLE +- name: proportion_timeout type: FLOAT -- mode: NULLABLE - name: proportion_abort + mode: NULLABLE +- name: proportion_abort type: FLOAT -- mode: NULLABLE - name: proportion_unreachable + mode: NULLABLE +- name: proportion_unreachable type: FLOAT -- mode: NULLABLE - name: proportion_terminated + mode: NULLABLE +- name: proportion_terminated type: FLOAT -- mode: NULLABLE - name: proportion_channel_open + mode: NULLABLE +- name: proportion_channel_open type: FLOAT -- mode: NULLABLE - name: avg_dns_success_time + mode: NULLABLE +- name: avg_dns_success_time type: FLOAT -- mode: NULLABLE - name: missing_dns_success + mode: NULLABLE +- name: missing_dns_success type: FLOAT -- mode: NULLABLE - name: avg_dns_failure_time + mode: NULLABLE +- name: avg_dns_failure_time type: FLOAT -- mode: NULLABLE - name: missing_dns_failure + mode: NULLABLE +- name: missing_dns_failure type: FLOAT -- mode: NULLABLE - name: count_dns_failure + mode: NULLABLE +- name: count_dns_failure type: FLOAT -- mode: NULLABLE - name: ssl_error_prop + mode: NULLABLE +- name: ssl_error_prop type: FLOAT -- mode: NULLABLE - name: avg_tls_handshake_time + mode: NULLABLE +- name: avg_tls_handshake_time type: FLOAT + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-04 16:40:15.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-04 16:43:39.000000000 +0000 @@ -45,7 +45,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -1565,7 +1565,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1575,7 +1575,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1656,7 +1656,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1666,7 +1666,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1747,7 +1747,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1757,7 +1757,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1838,7 +1838,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1848,7 +1848,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1990,7 +1990,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -2010,7 +2010,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.events_v1` + `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:14.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:02.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-04 16:48:39.000000000 +0000 @@ -1,7 +1,10 @@ fields: -- type: DATETIME - name: build_hour -- type: INTEGER - name: geckoview_major_version -- type: INTEGER - name: n_pings +- name: build_hour + type: DATETIME + mode: NULLABLE +- name: geckoview_major_version + type: INTEGER + mode: NULLABLE +- name: n_pings + type: INTEGER + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:15.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:03.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:14.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:04.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:40:15.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:04.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-04 16:48:48.000000000 +0000 @@ -1,9 +1,7 @@ fields: - name: postal_district_code type: STRING - mode: REQUIRED - description: One-character Canadian postal district code. + mode: NULLABLE - name: province_code type: STRING mode: NULLABLE - description: Two-character Canadian province/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-04 16:48:48.000000000 +0000 @@ -1,47 +1,28 @@ fields: - name: name - description: Official country name per ISO 3166 type: STRING - mode: REQUIRED + mode: NULLABLE - name: code - description: ISO 3166 alpha-2 country code type: STRING - mode: REQUIRED + mode: NULLABLE - name: code_3 - description: ISO 3166 alpha-3 country code type: STRING - mode: REQUIRED + mode: NULLABLE - name: region_name - description: Region name. These are based on the UN Statistics Division standard - country or area codes for statistical use (M49), but with the "Americas" region - split into "North America" and "South America". type: STRING - mode: REQUIRED + mode: NULLABLE - name: subregion_name - description: Sub-region name. These are based on UN Statistics Division standard - country or area codes for statistical use (M49), but with the "Latin America and the - Caribbean" and "Sub-Saharan Africa" sub-regions split into more specific - sub-regions. type: STRING - mode: REQUIRED + mode: NULLABLE - name: pocket_available_on_newtab - description: Whether Pocket is available on the newtab page in this country. Note - that Pocket might only be available in certain locales/languages within a country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: mozilla_vpn_available - description: Whether Mozilla VPN is available in this country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: sponsored_tiles_available_on_newtab - description: Whether sponsored tiles are available on the newtab page in this country. - Note that Pocket might only be available in certain locales/languages within a - country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: ads_value_tier - description: Lowercase label detailing the monetary value tier that Mozilla Ads - assign to that region based on market size and our existing products, e.g., tier - 1, tier 2, etc. type: STRING - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-04 16:48:48.000000000 +0000 @@ -1,10 +1,7 @@ fields: - name: name - description: An alias for a country's name (including misspellings and alternate - encodings). type: STRING - mode: REQUIRED + mode: NULLABLE - name: code - description: ISO 3166 alpha-2 country code type: STRING - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-04 16:48:48.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: incident + mode: NULLABLE +- name: incident type: STRING -- mode: NULLABLE - name: description + mode: NULLABLE +- name: description type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING -- mode: NULLABLE - name: product + mode: NULLABLE +- name: product type: STRING -- mode: NULLABLE - name: version + mode: NULLABLE +- name: version type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-04 16:48:48.000000000 +0000 @@ -1,27 +1,16 @@ fields: -- mode: NULLABLE - description: Hex value assigned to the TLS cipher, in format like "0x00,0x84"; note - some values are ranges or contain wildcards - name: value +- name: value type: STRING -- mode: NULLABLE - description: Human-readable name of the TLS cipher - name: description + mode: NULLABLE +- name: description type: STRING -- mode: NULLABLE - description: Any TLS cipher suite that is specified for use with DTLS MUST define - limits on the use of the associated AEAD function that preserves margins for both - confidentiality and integrity, as specified in [RFC-ietf-tls-dtls13-43] - name: dtls_ok + mode: NULLABLE +- name: dtls_ok type: BOOLEAN -- mode: NULLABLE - description: Whether the TLS cipher is recommended by the IETF. If an item is not - marked as "recommended", it does not necessarily mean that it is flawed; rather, - it indicates that the item either has not been through the IETF consensus process, - has limited applicability, or is intended only for specific use cases - name: recommended + mode: NULLABLE +- name: recommended type: BOOLEAN -- mode: NULLABLE - description: RFCs or associated reference material for the TLS cipher - name: reference + mode: NULLABLE +- name: reference type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-04 16:48:48.000000000 +0000 @@ -1,17 +1,13 @@ fields: - name: code_3 - description: ISO 639 alpha-3 language code. type: STRING - mode: REQUIRED + mode: NULLABLE - name: code_2 - description: ISO 639 alpha-2 language code (if any). type: STRING mode: NULLABLE - name: name - description: Language name. type: STRING - mode: REQUIRED + mode: NULLABLE - name: other_names - description: Other names for the language (if any). type: STRING mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-04 16:48:48.000000000 +0000 @@ -1,19 +1,19 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-04 16:48:48.000000000 +0000 @@ -1,25 +1,25 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: document_version + mode: NULLABLE +- name: document_version type: STRING -- mode: NULLABLE - name: path + mode: NULLABLE +- name: path type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-04 16:48:48.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: document_version + mode: NULLABLE +- name: document_version type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-04 16:48:48.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: path + mode: NULLABLE +- name: path type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-04 16:48:48.000000000 +0000 @@ -1,7 +1,7 @@ fields: - name: score type: NUMERIC - mode: REQUIRED + mode: NULLABLE - name: value type: NUMERIC - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-04 16:39:41.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-04 16:48:48.000000000 +0000 @@ -1,9 +1,7 @@ fields: - name: zip_code_prefix type: STRING - mode: REQUIRED - description: Three-digit US ZIP code prefix. + mode: NULLABLE - name: state_code type: STRING mode: NULLABLE - description: Two-character US state/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml 2024-06-04 16:40:15.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml 2024-06-04 16:48:52.000000000 +0000 @@ -2,21 +2,18 @@ - name: date type: DATE mode: NULLABLE - description: null - name: product type: STRING mode: NULLABLE - name: category type: STRING mode: NULLABLE - description: null - name: channel type: STRING mode: NULLABLE - name: build_number type: INTEGER mode: NULLABLE - description: null - name: release_date type: DATE mode: NULLABLE ```

Link to full diff

[dataops-ci-bot]

Integration report for "Update sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql"

sql.diff

Click to expand!

```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_braze.py /tmp/workspace/generated-sql/dags/bqetl_braze.py --- /tmp/workspace/main-generated-sql/dags/bqetl_braze.py 2024-06-04 16:42:49.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_braze.py 2024-06-04 16:55:40.000000000 +0000 @@ -238,21 +238,10 @@ email=["cbeck@mozilla.com", "leli@mozilla.com"], date_partition_parameter=None, depends_on_past=False, + parameters=["{'snapshot_interval': '1 HOUR'}"], sql_file_path="sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql", ) - braze_external__users_previous_day_snapshot__v2 = bigquery_etl_query( - task_id="braze_external__users_previous_day_snapshot__v2", - destination_table=None, - dataset_id="braze_external", - project_id="moz-fx-data-shared-prod", - owner="cbeck@mozilla.com", - email=["cbeck@mozilla.com", "leli@mozilla.com"], - date_partition_parameter=None, - depends_on_past=False, - sql_file_path="sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql", - ) - checks__fail_braze_derived__newsletters__v1 = bigquery_dq_check( task_id="checks__fail_braze_derived__newsletters__v1", source_table="newsletters_v1", @@ -480,7 +469,7 @@ ) braze_external__changed_users__v1.set_upstream( - braze_external__users_previous_day_snapshot__v2 + braze_external__users_previous_day_snapshot__v1 ) braze_external__changed_users__v1.set_upstream( diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_glean_usage.py /tmp/workspace/generated-sql/dags/bqetl_glean_usage.py --- /tmp/workspace/main-generated-sql/dags/bqetl_glean_usage.py 2024-06-04 16:42:49.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_glean_usage.py 2024-06-04 16:55:44.000000000 +0000 @@ -365,6 +365,20 @@ task_group=task_group_burnham, ) + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( + task_id="checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1", + source_table="baseline_clients_last_seen_v1", + dataset_id="accounts_cirrus_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=True, + owner="ascholtz@mozilla.com", + email=["ascholtz@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_accounts_cirrus, + ) + checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -1259,6 +1273,20 @@ task_group=task_group_pine, ) + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( + task_id="checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1", + source_table="baseline_clients_last_seen_v1", + dataset_id="accounts_cirrus_derived", + project_id="moz-fx-data-shared-prod", + is_dq_check_fail=False, + owner="ascholtz@mozilla.com", + email=["ascholtz@mozilla.com", "telemetry-alerts@mozilla.com"], + depends_on_past=False, + parameters=["submission_date:DATE:{{ds}}"], + retries=0, + task_group=task_group_accounts_cirrus, + ) + checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1 = bigquery_dq_check( task_id="checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1", source_table="baseline_clients_last_seen_v1", @@ -5100,6 +5128,18 @@ burnham_derived__metrics_clients_daily__v1 ) + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + accounts_cirrus_derived__baseline_clients_last_seen__v1 + ) + + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + checks__fail_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 + ) + checks__fail_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1.set_upstream( checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 ) @@ -5384,6 +5424,18 @@ pine_derived__baseline_clients_last_seen__v1 ) + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + accounts_cirrus_derived__baseline_clients_last_seen__v1 + ) + + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 + ) + + checks__warn_accounts_cirrus_derived__baseline_clients_last_seen__v1.set_upstream( + org_mozilla_ios_firefox_derived__baseline_clients_daily__v1 + ) + checks__warn_firefox_desktop_background_defaultagent_derived__baseline_clients_last_seen__v1.set_upstream( checks__fail_org_mozilla_ios_firefox_derived__baseline_clients_last_seen__v1 ) Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1: checks.sql Only in /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external: users_previous_day_snapshot_v2 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/checks.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/checks.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/checks.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/checks.sql 2024-06-04 16:42:23.000000000 +0000 @@ -0,0 +1,88 @@ +-- Generated via bigquery_etl.glean_usage + +#warn +{{ is_unique(["client_id"], where="submission_date = @submission_date") }} + +#warn +{{ min_row_count(1, where="submission_date = @submission_date") }} + +#warn +{{ not_null([ + "submission_date", + "client_id", + "sample_id", + "first_seen_date", + "days_seen_bits", + "days_active_bits", + "days_created_profile_bits", + "days_seen_session_start_bits", + "days_seen_session_end_bits" + ], where="submission_date = @submission_date") }} + +#warn +SELECT + IF( + COUNTIF(normalized_channel NOT IN ("nightly", "aurora", "release", "Other", "beta", "esr")) > 0, + ERROR("Unexpected values for field normalized_channel detected."), + NULL + ) +FROM + `{{ project_id }}.{{ dataset_id }}.{{ table_name }}` +WHERE + submission_date = @submission_date; + +#warn +{{ matches_pattern(column="country", pattern="^[A-Z]{2}$", where="submission_date = @submission_date") }} + +#warn +{{ value_length(column="client_id", expected_length=36, where="submission_date = @submission_date") }} + +#fail +WITH daily AS ( + SELECT + submission_date, + COUNT(DISTINCT client_id) AS client_count + FROM + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_daily_v1` + WHERE + submission_date = @submission_date + AND sample_id IS NOT NULL + GROUP BY + submission_date +), +last_seen AS ( + SELECT + submission_date, + COUNT(DISTINCT client_id) AS client_count + FROM + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_last_seen_v1` + WHERE + submission_date = @submission_date + AND mozfun.bits28.days_since_seen(days_seen_bits) = 0 + GROUP BY + submission_date +), +check_results AS ( + SELECT + 1 - (last_seen.client_count / daily.client_count) AS difference_perc + FROM + daily + LEFT JOIN + last_seen + USING (submission_date) +) +SELECT + IF( + ABS((SELECT difference_perc FROM check_results)) > 0.001, + ERROR( + CONCAT( + "Results don't match by > 1%, baseline_clients_daily table has ", + STRING(((SELECT submission_date FROM daily))), + ": ", + ABS((SELECT client_count FROM daily)), + ". baseline_clients_last_seen has ", + IFNULL(((SELECT client_count FROM last_seen)), 0) + ) + ), + NULL + ); diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/metadata.yaml 2024-06-04 16:39:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/accounts_cirrus_derived/baseline_clients_last_seen_v1/metadata.yaml 2024-06-04 16:52:49.000000000 +0000 @@ -40,6 +40,10 @@ members: - workgroup:mozilla-confidential references: + checks.sql: + - .. + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_daily_v1 + - moz-fx-data-shared-prod.org_mozilla_ios_firefox_derived.baseline_clients_last_seen_v1 query.sql: - accounts_cirrus_derived.baseline_clients_daily_v1 - accounts_cirrus_derived.baseline_clients_last_seen_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:39:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:06.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.events_v1` + `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml 2024-06-04 16:40:49.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/metadata.yaml 2024-06-04 16:52:46.000000000 +0000 @@ -33,4 +33,4 @@ - .. query.sql: - moz-fx-data-shared-prod.braze_derived.users_v1 - - moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v2 + - moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql 2024-06-04 16:39:07.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_v1/query.sql 2024-06-04 16:40:11.000000000 +0000 @@ -38,7 +38,7 @@ FROM `moz-fx-data-shared-prod.braze_derived.users_v1` current_users FULL OUTER JOIN - `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v2` previous_users --change back to v1 after run tomorrow + `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1` previous_users --change back to v1 after run tomorrow ON current_users.external_id = previous_users.external_id WHERE current_users.external_id IS NULL -- deleted rows diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml 2024-06-04 16:40:49.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/metadata.yaml 2024-06-04 16:52:46.000000000 +0000 @@ -18,6 +18,8 @@ scheduling: dag_name: bqetl_braze date_partition_parameter: null + parameters: + - snapshot_interval: 1 HOUR bigquery: time_partitioning: null range_partitioning: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql 2024-06-04 16:39:07.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v1/script.sql 2024-06-04 16:40:11.000000000 +0000 @@ -2,7 +2,5 @@ CREATE SNAPSHOT TABLE `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v1` CLONE `moz-fx-data-shared-prod.braze_derived.users_v1` FOR SYSTEM_TIME AS OF TIMESTAMP_SUB( CURRENT_TIMESTAMP(), - INTERVAL 12 HOUR + INTERVAL snapshot_interval HOUR ); - - -- Chose 12 hours here because it's split between when the job last ran and now (yesterday). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/metadata.yaml 2024-06-04 16:40:49.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 @@ -1,34 +0,0 @@ -friendly_name: Braze User Profiles Previous Day Snapshot v2 -description: |- - This is a snapshot of the users table at the state it was in before - the most recent DAG run. Since the user profiles table updates once a day at - 10AM UTC, we can update this snapshot looking 12 hours back (yesterday). - - This snapshot allows us to compare changed values in order to generate the - JSON payloads for Braze's cloud data ingestion tool. - - See https://mozilla-hub.atlassian.net/browse/DENG-3182 -owners: -- cbeck@mozilla.com -labels: - schedule: daily - owner: cbeck - dag: bqetl_braze - owner1: cbeck -scheduling: - dag_name: bqetl_braze - date_partition_parameter: null -bigquery: - time_partitioning: null - range_partitioning: null - clustering: null -workgroup_access: -- role: roles/bigquery.dataViewer - members: - - workgroup:braze - - workgroup:braze/ingestion-mozilla-dev - - workgroup:braze/ingestion-firefox -- role: roles/bigquery.dataEditor - members: - - workgroup:braze/writers -references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql 2024-06-04 16:39:07.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/users_previous_day_snapshot_v2/script.sql 1970-01-01 00:00:00.000000000 +0000 @@ -1,8 +0,0 @@ -DROP SNAPSHOT TABLE `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v2`; - -CREATE SNAPSHOT TABLE `moz-fx-data-shared-prod.braze_external.users_previous_day_snapshot_v2` CLONE `moz-fx-data-shared-prod.braze_derived.users_v1` FOR SYSTEM_TIME AS OF TIMESTAMP_SUB( - CURRENT_TIMESTAMP(), - INTERVAL 1 HOUR -); - - -- Chose 12 hours here because it's split between when the job last ran and now (yesterday). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-04 16:49:18.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: source + mode: NULLABLE +- name: source type: STRING -- mode: NULLABLE - name: event_type + mode: NULLABLE +- name: event_type type: STRING -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: subdivision1 + mode: NULLABLE +- name: subdivision1 type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: event_count + mode: NULLABLE +- name: event_count type: INTEGER -- mode: NULLABLE - name: user_count + mode: NULLABLE +- name: user_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-04 16:49:18.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: impression_count + mode: NULLABLE +- name: impression_count type: INTEGER -- mode: NULLABLE - name: click_count + mode: NULLABLE +- name: click_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-04 16:47:28.000000000 +0000 @@ -26,6 +26,9 @@ - name: adjust_network type: STRING mode: NULLABLE +- name: install_source + type: STRING + mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-04 16:47:27.000000000 +0000 @@ -48,6 +48,10 @@ description: 'The type of source of a client installation. ' +- name: install_source + type: STRING + mode: NULLABLE + description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:39:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:07.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:39:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:07.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -80,7 +80,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-04 16:50:01.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: country +- name: country type: STRING -- mode: NULLABLE - name: city + mode: NULLABLE +- name: city type: STRING -- mode: NULLABLE - name: datetime + mode: NULLABLE +- name: datetime type: TIMESTAMP -- mode: NULLABLE - name: proportion_undefined + mode: NULLABLE +- name: proportion_undefined type: FLOAT -- mode: NULLABLE - name: proportion_timeout + mode: NULLABLE +- name: proportion_timeout type: FLOAT -- mode: NULLABLE - name: proportion_abort + mode: NULLABLE +- name: proportion_abort type: FLOAT -- mode: NULLABLE - name: proportion_unreachable + mode: NULLABLE +- name: proportion_unreachable type: FLOAT -- mode: NULLABLE - name: proportion_terminated + mode: NULLABLE +- name: proportion_terminated type: FLOAT -- mode: NULLABLE - name: proportion_channel_open + mode: NULLABLE +- name: proportion_channel_open type: FLOAT -- mode: NULLABLE - name: avg_dns_success_time + mode: NULLABLE +- name: avg_dns_success_time type: FLOAT -- mode: NULLABLE - name: missing_dns_success + mode: NULLABLE +- name: missing_dns_success type: FLOAT -- mode: NULLABLE - name: avg_dns_failure_time + mode: NULLABLE +- name: avg_dns_failure_time type: FLOAT -- mode: NULLABLE - name: missing_dns_failure + mode: NULLABLE +- name: missing_dns_failure type: FLOAT -- mode: NULLABLE - name: count_dns_failure + mode: NULLABLE +- name: count_dns_failure type: FLOAT -- mode: NULLABLE - name: ssl_error_prop + mode: NULLABLE +- name: ssl_error_prop type: FLOAT -- mode: NULLABLE - name: avg_tls_handshake_time + mode: NULLABLE +- name: avg_tls_handshake_time type: FLOAT + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-04 16:39:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-04 16:43:41.000000000 +0000 @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -75,7 +75,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1555,7 +1555,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1565,7 +1565,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1646,7 +1646,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1656,7 +1656,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1737,7 +1737,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1747,7 +1747,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1828,7 +1828,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1838,7 +1838,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1990,7 +1990,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -2010,7 +2010,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.events_v1` + `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2162,7 +2162,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -2172,7 +2172,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:39:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:06.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.main_v1` + `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.main_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-04 16:49:28.000000000 +0000 @@ -1,7 +1,10 @@ fields: -- type: DATETIME - name: build_hour -- type: INTEGER - name: geckoview_major_version -- type: INTEGER - name: n_pings +- name: build_hour + type: DATETIME + mode: NULLABLE +- name: geckoview_major_version + type: INTEGER + mode: NULLABLE +- name: n_pings + type: INTEGER + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:39:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:08.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:39:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:09.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:39:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-04 16:42:09.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-04 16:50:10.000000000 +0000 @@ -1,9 +1,7 @@ fields: - name: postal_district_code type: STRING - mode: REQUIRED - description: One-character Canadian postal district code. + mode: NULLABLE - name: province_code type: STRING mode: NULLABLE - description: Two-character Canadian province/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-04 16:50:10.000000000 +0000 @@ -1,47 +1,28 @@ fields: - name: name - description: Official country name per ISO 3166 type: STRING - mode: REQUIRED + mode: NULLABLE - name: code - description: ISO 3166 alpha-2 country code type: STRING - mode: REQUIRED + mode: NULLABLE - name: code_3 - description: ISO 3166 alpha-3 country code type: STRING - mode: REQUIRED + mode: NULLABLE - name: region_name - description: Region name. These are based on the UN Statistics Division standard - country or area codes for statistical use (M49), but with the "Americas" region - split into "North America" and "South America". type: STRING - mode: REQUIRED + mode: NULLABLE - name: subregion_name - description: Sub-region name. These are based on UN Statistics Division standard - country or area codes for statistical use (M49), but with the "Latin America and the - Caribbean" and "Sub-Saharan Africa" sub-regions split into more specific - sub-regions. type: STRING - mode: REQUIRED + mode: NULLABLE - name: pocket_available_on_newtab - description: Whether Pocket is available on the newtab page in this country. Note - that Pocket might only be available in certain locales/languages within a country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: mozilla_vpn_available - description: Whether Mozilla VPN is available in this country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: sponsored_tiles_available_on_newtab - description: Whether sponsored tiles are available on the newtab page in this country. - Note that Pocket might only be available in certain locales/languages within a - country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: ads_value_tier - description: Lowercase label detailing the monetary value tier that Mozilla Ads - assign to that region based on market size and our existing products, e.g., tier - 1, tier 2, etc. type: STRING - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-04 16:50:10.000000000 +0000 @@ -1,10 +1,7 @@ fields: - name: name - description: An alias for a country's name (including misspellings and alternate - encodings). type: STRING - mode: REQUIRED + mode: NULLABLE - name: code - description: ISO 3166 alpha-2 country code type: STRING - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-04 16:50:11.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: incident + mode: NULLABLE +- name: incident type: STRING -- mode: NULLABLE - name: description + mode: NULLABLE +- name: description type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING -- mode: NULLABLE - name: product + mode: NULLABLE +- name: product type: STRING -- mode: NULLABLE - name: version + mode: NULLABLE +- name: version type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-04 16:50:10.000000000 +0000 @@ -1,27 +1,16 @@ fields: -- mode: NULLABLE - description: Hex value assigned to the TLS cipher, in format like "0x00,0x84"; note - some values are ranges or contain wildcards - name: value +- name: value type: STRING -- mode: NULLABLE - description: Human-readable name of the TLS cipher - name: description + mode: NULLABLE +- name: description type: STRING -- mode: NULLABLE - description: Any TLS cipher suite that is specified for use with DTLS MUST define - limits on the use of the associated AEAD function that preserves margins for both - confidentiality and integrity, as specified in [RFC-ietf-tls-dtls13-43] - name: dtls_ok + mode: NULLABLE +- name: dtls_ok type: BOOLEAN -- mode: NULLABLE - description: Whether the TLS cipher is recommended by the IETF. If an item is not - marked as "recommended", it does not necessarily mean that it is flawed; rather, - it indicates that the item either has not been through the IETF consensus process, - has limited applicability, or is intended only for specific use cases - name: recommended + mode: NULLABLE +- name: recommended type: BOOLEAN -- mode: NULLABLE - description: RFCs or associated reference material for the TLS cipher - name: reference + mode: NULLABLE +- name: reference type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-04 16:50:11.000000000 +0000 @@ -1,17 +1,13 @@ fields: - name: code_3 - description: ISO 639 alpha-3 language code. type: STRING - mode: REQUIRED + mode: NULLABLE - name: code_2 - description: ISO 639 alpha-2 language code (if any). type: STRING mode: NULLABLE - name: name - description: Language name. type: STRING - mode: REQUIRED + mode: NULLABLE - name: other_names - description: Other names for the language (if any). type: STRING mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-04 16:50:10.000000000 +0000 @@ -1,19 +1,19 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-04 16:50:10.000000000 +0000 @@ -1,25 +1,25 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: document_version + mode: NULLABLE +- name: document_version type: STRING -- mode: NULLABLE - name: path + mode: NULLABLE +- name: path type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-04 16:50:10.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: document_version + mode: NULLABLE +- name: document_version type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-04 16:50:11.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: path + mode: NULLABLE +- name: path type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-04 16:50:10.000000000 +0000 @@ -1,7 +1,7 @@ fields: - name: score type: NUMERIC - mode: REQUIRED + mode: NULLABLE - name: value type: NUMERIC - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-04 16:39:06.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-04 16:50:10.000000000 +0000 @@ -1,9 +1,7 @@ fields: - name: zip_code_prefix type: STRING - mode: REQUIRED - description: Three-digit US ZIP code prefix. + mode: NULLABLE - name: state_code type: STRING mode: NULLABLE - description: Two-character US state/territory code (if any). ```

Link to full diff