search

mozilla / bigquery-etl

Bigquery ETL
https://mozilla.github.io/bigquery-etl
Mozilla Public License 2.0
241 stars 98 forks source link

GROWTH-143 Productionalize Census feed #5763

Closed kwindau closed 4 weeks ago

kwindau commented 4 weeks ago

Checklist for reviewer:

For modifications to schemas in restricted namespaces (see CODEOWNERS):

┆Issue is synchronized with this Jira Task

dataops-ci-bot commented 4 weeks ago

Integration report for "Add bqetl_census_feed to dags.yaml"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 18:20:19.000000000 +0000 @@ -0,0 +1,80 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v1 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v1", + external_dag_id="bqetl_mozilla_org_derived", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v1", + execution_delta=datetime.timedelta(seconds=54000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table="ga_desktop_conversions_v1", + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v1 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_mozilla_org_derived.py /tmp/workspace/generated-sql/dags/bqetl_mozilla_org_derived.py --- /tmp/workspace/main-generated-sql/dags/bqetl_mozilla_org_derived.py 2024-06-06 18:19:20.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_mozilla_org_derived.py 2024-06-06 18:20:19.000000000 +0000 @@ -147,6 +147,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v1_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v1_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v1", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=32400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v1_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v1 + ) + checks__fail_stub_attribution_service_derived__dl_token_ga_attribution_lookup__v1 = bigquery_dq_check( task_id="checks__fail_stub_attribution_service_derived__dl_token_ga_attribution_lookup__v1", source_table="dl_token_ga_attribution_lookup_v1", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 18:17:32.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 18:15:43.000000000 +0000 @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 18:17:31.000000000 +0000 @@ -0,0 +1,26 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed +bigquery: + time_partitioning: + type: day + field: activity_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 18:15:43.000000000 +0000 @@ -0,0 +1,29 @@ +SELECT + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_date, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod`.mozilla_org_derived.gclid_conversions_v1 UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date >= DATE_SUB(CURRENT_DATE, INTERVAL 2 DAY) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 18:15:43.000000000 +0000 @@ -0,0 +1,13 @@ +fields: +- name: activity_date + mode: NULLABLE + type: DATE + description: Activity Date +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name ```

Link to full diff

dataops-ci-bot commented 4 weeks ago

Integration report for "Switch to using new v2 table of gclid conversions"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 18:35:33.000000000 +0000 @@ -0,0 +1,80 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_google_analytics_derived_ga4", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_delta=datetime.timedelta(seconds=18000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table="ga_desktop_conversions_v1", + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 18:34:22.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 18:35:33.000000000 +0000 @@ -153,6 +153,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v2_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v2_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=68400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v2_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) + checks__warn_ga_derived__blogs_goals__v2 = bigquery_dq_check( task_id="checks__warn_ga_derived__blogs_goals__v2", source_table="blogs_goals_v2", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql 2024-06-06 18:30:38.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql 2024-06-06 18:30:17.000000000 +0000 @@ -42,34 +42,20 @@ jobs AS ( -- https://cloud.google.com/bigquery/docs/information-schema-jobs SELECT - creation_time, - start_time, - end_time, - state, - error_result, - job_id, - project_id, - total_bytes_processed AS bytes_complete, - total_slot_ms AS slot_ms, + * FROM `moz-fx-data-shredder.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT` UNION ALL SELECT - creation_time, - start_time, - end_time, - state, - error_result, - job_id, - project_id, - total_bytes_processed AS bytes_complete, - total_slot_ms AS slot_ms, + * FROM `moz-fx-data-bq-batch-prod.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT` ), successful_jobs AS ( SELECT - * + *, + total_bytes_processed AS bytes_complete, + total_slot_ms AS slot_ms, FROM jobs WHERE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 18:32:10.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 18:30:17.000000000 +0000 @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 18:32:16.000000000 +0000 @@ -0,0 +1,26 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed +bigquery: + time_partitioning: + type: day + field: activity_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 18:30:17.000000000 +0000 @@ -0,0 +1,29 @@ +SELECT + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_date, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.gclid_conversions_v2` UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date >= DATE_SUB(CURRENT_DATE, INTERVAL 2 DAY) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 18:30:17.000000000 +0000 @@ -0,0 +1,13 @@ +fields: +- name: activity_date + mode: NULLABLE + type: DATE + description: Activity Date +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name ```

Link to full diff

dataops-ci-bot commented 4 weeks ago

Integration report for "Add bqetl_census_feed to dags.yaml"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 18:46:07.000000000 +0000 @@ -0,0 +1,80 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v1 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v1", + external_dag_id="bqetl_mozilla_org_derived", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v1", + execution_delta=datetime.timedelta(seconds=54000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table="ga_desktop_conversions_v1", + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v1 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_mozilla_org_derived.py /tmp/workspace/generated-sql/dags/bqetl_mozilla_org_derived.py --- /tmp/workspace/main-generated-sql/dags/bqetl_mozilla_org_derived.py 2024-06-06 18:44:51.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_mozilla_org_derived.py 2024-06-06 18:46:07.000000000 +0000 @@ -147,6 +147,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v1_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v1_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v1", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v1", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=32400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v1_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v1 + ) + checks__fail_stub_attribution_service_derived__dl_token_ga_attribution_lookup__v1 = bigquery_dq_check( task_id="checks__fail_stub_attribution_service_derived__dl_token_ga_attribution_lookup__v1", source_table="dl_token_ga_attribution_lookup_v1", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql 2024-06-06 18:41:28.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql 2024-06-06 18:41:29.000000000 +0000 @@ -42,34 +42,20 @@ jobs AS ( -- https://cloud.google.com/bigquery/docs/information-schema-jobs SELECT - creation_time, - start_time, - end_time, - state, - error_result, - job_id, - project_id, - total_bytes_processed AS bytes_complete, - total_slot_ms AS slot_ms, + * FROM `moz-fx-data-shredder.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT` UNION ALL SELECT - creation_time, - start_time, - end_time, - state, - error_result, - job_id, - project_id, - total_bytes_processed AS bytes_complete, - total_slot_ms AS slot_ms, + * FROM `moz-fx-data-bq-batch-prod.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT` ), successful_jobs AS ( SELECT - * + *, + total_bytes_processed AS bytes_complete, + total_slot_ms AS slot_ms, FROM jobs WHERE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 18:43:16.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 18:41:29.000000000 +0000 @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 18:43:12.000000000 +0000 @@ -0,0 +1,26 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed +bigquery: + time_partitioning: + type: day + field: activity_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 18:41:29.000000000 +0000 @@ -0,0 +1,29 @@ +SELECT + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_date, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod`.mozilla_org_derived.gclid_conversions_v1 UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date >= DATE_SUB(CURRENT_DATE, INTERVAL 2 DAY) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 18:41:29.000000000 +0000 @@ -0,0 +1,13 @@ +fields: +- name: activity_date + mode: NULLABLE + type: DATE + description: Activity Date +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name ```

Link to full diff

dataops-ci-bot commented 4 weeks ago

Integration report for "Switch to using new v2 table of gclid conversions"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 18:46:46.000000000 +0000 @@ -0,0 +1,80 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_google_analytics_derived_ga4", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_delta=datetime.timedelta(seconds=18000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table="ga_desktop_conversions_v1", + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter="submission_date", + depends_on_past=False, + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 18:45:48.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 18:46:47.000000000 +0000 @@ -153,6 +153,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v2_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v2_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=68400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v2_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) + checks__warn_ga_derived__blogs_goals__v2 = bigquery_dq_check( task_id="checks__warn_ga_derived__blogs_goals__v2", source_table="blogs_goals_v2", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql 2024-06-06 18:41:53.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql 2024-06-06 18:41:49.000000000 +0000 @@ -42,34 +42,20 @@ jobs AS ( -- https://cloud.google.com/bigquery/docs/information-schema-jobs SELECT - creation_time, - start_time, - end_time, - state, - error_result, - job_id, - project_id, - total_bytes_processed AS bytes_complete, - total_slot_ms AS slot_ms, + * FROM `moz-fx-data-shredder.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT` UNION ALL SELECT - creation_time, - start_time, - end_time, - state, - error_result, - job_id, - project_id, - total_bytes_processed AS bytes_complete, - total_slot_ms AS slot_ms, + * FROM `moz-fx-data-bq-batch-prod.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT` ), successful_jobs AS ( SELECT - * + *, + total_bytes_processed AS bytes_complete, + total_slot_ms AS slot_ms, FROM jobs WHERE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 18:43:36.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 18:41:49.000000000 +0000 @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 18:43:36.000000000 +0000 @@ -0,0 +1,26 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed +bigquery: + time_partitioning: + type: day + field: activity_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 18:41:49.000000000 +0000 @@ -0,0 +1,29 @@ +SELECT + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_date, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.gclid_conversions_v2` UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date >= DATE_SUB(CURRENT_DATE, INTERVAL 2 DAY) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 18:41:49.000000000 +0000 @@ -0,0 +1,13 @@ +fields: +- name: activity_date + mode: NULLABLE + type: DATE + description: Activity Date +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name ```

Link to full diff

dataops-ci-bot commented 4 weeks ago

Integration report for "GROWTH-143 add activity date as incremental parameter"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 19:24:12.000000000 +0000 @@ -0,0 +1,82 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_google_analytics_derived_ga4", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_delta=datetime.timedelta(seconds=18000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table='ga_desktop_conversions_v1${{ macros.ds_format(macros.ds_add(ds, -2), "%Y-%m-%d", "%Y%m%d") }}', + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + parameters=["activity_date:DATE:{{macros.ds_add(ds, -2)}}"] + + ["submission_date:DATE:{{ds}}"], + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 19:37:06.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 19:24:12.000000000 +0000 @@ -153,6 +153,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v2_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v2_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=68400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v2_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) + checks__warn_ga_derived__blogs_goals__v2 = bigquery_dq_check( task_id="checks__warn_ga_derived__blogs_goals__v2", source_table="blogs_goals_v2", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/topsites_impression: schema.yaml diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:34:21.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:20:19.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.events_v1` + `moz-fx-data-shared-prod.bedrock_live.interaction_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-06 19:34:20.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-06 19:19:28.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- name: submission_date +- mode: NULLABLE + name: submission_date type: DATE - mode: NULLABLE -- name: source +- mode: NULLABLE + name: source type: STRING - mode: NULLABLE -- name: event_type +- mode: NULLABLE + name: event_type type: STRING - mode: NULLABLE -- name: form_factor +- mode: NULLABLE + name: form_factor type: STRING - mode: NULLABLE -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: subdivision1 +- mode: NULLABLE + name: subdivision1 type: STRING - mode: NULLABLE -- name: advertiser +- mode: NULLABLE + name: advertiser type: STRING - mode: NULLABLE -- name: release_channel +- mode: NULLABLE + name: release_channel type: STRING - mode: NULLABLE -- name: position +- mode: NULLABLE + name: position type: INTEGER - mode: NULLABLE -- name: provider +- mode: NULLABLE + name: provider type: STRING - mode: NULLABLE -- name: match_type +- mode: NULLABLE + name: match_type type: STRING - mode: NULLABLE -- name: normalized_os +- mode: NULLABLE + name: normalized_os type: STRING - mode: NULLABLE -- name: suggest_data_sharing_enabled +- mode: NULLABLE + name: suggest_data_sharing_enabled type: BOOLEAN - mode: NULLABLE -- name: event_count +- mode: NULLABLE + name: event_count type: INTEGER - mode: NULLABLE -- name: user_count +- mode: NULLABLE + name: user_count type: INTEGER - mode: NULLABLE -- name: query_type +- mode: NULLABLE + name: query_type type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-06 19:34:20.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-06 19:19:28.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- name: submission_date +- mode: NULLABLE + name: submission_date type: DATE - mode: NULLABLE -- name: form_factor +- mode: NULLABLE + name: form_factor type: STRING - mode: NULLABLE -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: advertiser +- mode: NULLABLE + name: advertiser type: STRING - mode: NULLABLE -- name: normalized_os +- mode: NULLABLE + name: normalized_os type: STRING - mode: NULLABLE -- name: release_channel +- mode: NULLABLE + name: release_channel type: STRING - mode: NULLABLE -- name: position +- mode: NULLABLE + name: position type: INTEGER - mode: NULLABLE -- name: provider +- mode: NULLABLE + name: provider type: STRING - mode: NULLABLE -- name: match_type +- mode: NULLABLE + name: match_type type: STRING - mode: NULLABLE -- name: suggest_data_sharing_enabled +- mode: NULLABLE + name: suggest_data_sharing_enabled type: BOOLEAN - mode: NULLABLE -- name: impression_count +- mode: NULLABLE + name: impression_count type: INTEGER - mode: NULLABLE -- name: click_count +- mode: NULLABLE + name: click_count type: INTEGER - mode: NULLABLE -- name: query_type +- mode: NULLABLE + name: query_type type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-06 19:34:21.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-06 19:19:28.000000000 +0000 @@ -26,9 +26,6 @@ - name: adjust_network type: STRING mode: NULLABLE -- name: install_source - type: STRING - mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-06 19:34:21.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-06 19:19:28.000000000 +0000 @@ -48,10 +48,6 @@ description: 'The type of source of a client installation. ' -- name: install_source - type: STRING - mode: NULLABLE - description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:34:20.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:20:20.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:34:20.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:20:19.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/query.sql 2024-06-06 19:34:20.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/query.sql 2024-06-06 19:20:19.000000000 +0000 @@ -69,13 +69,10 @@ COALESCE(metrics.boolean.urlbar_pref_suggest_nonsponsored, FALSE) AS pref_fx_suggestions, mozfun.map.get_key(extra, "engagement_type") AS engagement_type, mozfun.map.get_key(extra, "interaction") AS interaction, - SAFE_CAST(mozfun.map.get_key(extra, "n_chars") AS int) AS num_chars_typed, - SAFE_CAST(mozfun.map.get_key(extra, "n_results") AS int) AS num_total_results, + CAST(mozfun.map.get_key(extra, "n_chars") AS int) AS num_chars_typed, + CAST(mozfun.map.get_key(extra, "n_results") AS int) AS num_total_results, --If 0, then no result was selected. - NULLIF( - SAFE_CAST(mozfun.map.get_key(extra, "selected_position") AS int), - 0 - ) AS selected_position, + NULLIF(CAST(mozfun.map.get_key(extra, "selected_position") AS int), 0) AS selected_position, mozfun.map.get_key(extra, "selected_result") AS selected_result, enumerated_array( SPLIT(mozfun.map.get_key(extra, "results"), ','), diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-06 19:34:21.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-06 19:19:28.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: city +- mode: NULLABLE + name: city type: STRING - mode: NULLABLE -- name: datetime +- mode: NULLABLE + name: datetime type: TIMESTAMP - mode: NULLABLE -- name: proportion_undefined +- mode: NULLABLE + name: proportion_undefined type: FLOAT - mode: NULLABLE -- name: proportion_timeout +- mode: NULLABLE + name: proportion_timeout type: FLOAT - mode: NULLABLE -- name: proportion_abort +- mode: NULLABLE + name: proportion_abort type: FLOAT - mode: NULLABLE -- name: proportion_unreachable +- mode: NULLABLE + name: proportion_unreachable type: FLOAT - mode: NULLABLE -- name: proportion_terminated +- mode: NULLABLE + name: proportion_terminated type: FLOAT - mode: NULLABLE -- name: proportion_channel_open +- mode: NULLABLE + name: proportion_channel_open type: FLOAT - mode: NULLABLE -- name: avg_dns_success_time +- mode: NULLABLE + name: avg_dns_success_time type: FLOAT - mode: NULLABLE -- name: missing_dns_success +- mode: NULLABLE + name: missing_dns_success type: FLOAT - mode: NULLABLE -- name: avg_dns_failure_time +- mode: NULLABLE + name: avg_dns_failure_time type: FLOAT - mode: NULLABLE -- name: missing_dns_failure +- mode: NULLABLE + name: missing_dns_failure type: FLOAT - mode: NULLABLE -- name: count_dns_failure +- mode: NULLABLE + name: count_dns_failure type: FLOAT - mode: NULLABLE -- name: ssl_error_prop +- mode: NULLABLE + name: ssl_error_prop type: FLOAT - mode: NULLABLE -- name: avg_tls_handshake_time +- mode: NULLABLE + name: avg_tls_handshake_time type: FLOAT - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql 2024-06-06 19:34:21.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql 2024-06-06 19:19:28.000000000 +0000 @@ -42,34 +42,20 @@ jobs AS ( -- https://cloud.google.com/bigquery/docs/information-schema-jobs SELECT - creation_time, - start_time, - end_time, - state, - error_result, - job_id, - project_id, - total_bytes_processed AS bytes_complete, - total_slot_ms AS slot_ms, + * FROM `moz-fx-data-shredder.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT` UNION ALL SELECT - creation_time, - start_time, - end_time, - state, - error_result, - job_id, - project_id, - total_bytes_processed AS bytes_complete, - total_slot_ms AS slot_ms, + * FROM `moz-fx-data-bq-batch-prod.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT` ), successful_jobs AS ( SELECT - * + *, + total_bytes_processed AS bytes_complete, + total_slot_ms AS slot_ms, FROM jobs WHERE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-06 19:34:20.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-06 19:20:19.000000000 +0000 @@ -45,7 +45,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -572,7 +572,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -582,7 +582,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -592,7 +592,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -663,7 +663,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -673,7 +673,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -683,7 +683,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -754,7 +754,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -764,7 +764,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -774,7 +774,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1555,7 +1555,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1575,7 +1575,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1646,7 +1646,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1666,7 +1666,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1737,7 +1737,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1757,7 +1757,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1828,7 +1828,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1848,7 +1848,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1990,7 +1990,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2000,7 +2000,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -2010,7 +2010,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.events_v1` + `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2081,7 +2081,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` + `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` UNION ALL SELECT submission_timestamp, @@ -2091,7 +2091,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` + `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2162,7 +2162,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -2172,7 +2172,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 19:21:18.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 19:19:28.000000000 +0000 @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 19:21:16.000000000 +0000 @@ -0,0 +1,30 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed + date_partition_parameter: activity_date + date_partition_offset: -2 + parameters: + - submission_date:DATE:{{ds}} +bigquery: + time_partitioning: + type: day + field: activity_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 19:19:28.000000000 +0000 @@ -0,0 +1,29 @@ +SELECT + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_date, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.gclid_conversions_v2` UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date = DATE_SUB(@activity_date, INTERVAL 2 DAY) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 19:19:28.000000000 +0000 @@ -0,0 +1,13 @@ +fields: +- name: activity_date + mode: NULLABLE + type: DATE + description: Activity Date +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:34:21.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:20:20.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.main_v1` + `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-06 19:34:20.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-06 19:19:28.000000000 +0000 @@ -1,13 +1,7 @@ fields: -- name: build_hour - type: DATETIME - mode: NULLABLE - description: null -- name: geckoview_major_version - type: INTEGER - mode: NULLABLE - description: null -- name: n_pings - type: INTEGER - mode: NULLABLE - description: null +- type: DATETIME + name: build_hour +- type: INTEGER + name: geckoview_major_version +- type: INTEGER + name: n_pings diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/topsites_impression/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/topsites_impression/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/topsites_impression/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/topsites_impression/schema.yaml 2024-06-06 19:20:20.000000000 +0000 @@ -0,0 +1,449 @@ +fields: +- name: additional_properties + type: STRING + mode: NULLABLE + description: A JSON string containing any payload properties not present in the + schema +- name: client_info + type: RECORD + mode: NULLABLE + fields: + - name: android_sdk_version + type: STRING + mode: NULLABLE + description: The optional Android specific SDK version of the software running + on this hardware device. + - name: app_build + type: STRING + mode: NULLABLE + description: The build identifier generated by the CI system (e.g. "1234/A"). + For language bindings that provide automatic detection for this value, (e.g. + Android/Kotlin), in the unlikely event that the build identifier can not be + retrieved from the OS, it is set to "inaccessible". For other language bindings, + if the value was not provided through configuration, this metric gets set to + `Unknown`. + - name: app_channel + type: STRING + mode: NULLABLE + description: The channel the application is being distributed on. + - name: app_display_version + type: STRING + mode: NULLABLE + description: The user visible version string (e.g. "1.0.3"). In the unlikely + event that the display version can not be retrieved, it is set to "inaccessible". + - name: architecture + type: STRING + mode: NULLABLE + description: The architecture of the device, (e.g. "arm", "x86"). + - name: build_date + type: STRING + mode: NULLABLE + description: The date & time the application was built + - name: client_id + type: STRING + mode: NULLABLE + description: A UUID uniquely identifying the client. + - name: device_manufacturer + type: STRING + mode: NULLABLE + description: The manufacturer of the device the application is running on. Not + set if the device manufacturer can't be determined (e.g. on Desktop). + - name: device_model + type: STRING + mode: NULLABLE + description: The model of the device the application is running on. On Android, + this is Build.MODEL, the user-visible marketing name, like "Pixel 2 XL". Not + set if the device model can't be determined (e.g. on Desktop). + - name: first_run_date + type: STRING + mode: NULLABLE + description: The date of the first run of the application. + - name: locale + type: STRING + mode: NULLABLE + description: The locale of the application during initialization (e.g. "es-ES"). + If the locale can't be determined on the system, the value is ["und"](https://unicode.org/reports/tr35/#Unknown_or_Invalid_Identifiers), + to indicate "undetermined". + - name: os + type: STRING + mode: NULLABLE + description: 'The name of the operating system. Possible values: Android, iOS, + Linux, Darwin, Windows, FreeBSD, NetBSD, OpenBSD, Solaris, unknown' + - name: os_version + type: STRING + mode: NULLABLE + description: The user-visible version of the operating system (e.g. "1.2.3"). + If the version detection fails, this metric gets set to `Unknown`. + - name: telemetry_sdk_build + type: STRING + mode: NULLABLE + description: The version of the Glean SDK + - name: windows_build_number + type: INTEGER + mode: NULLABLE + description: The optional Windows build number, reported by Windows (e.g. 22000) + and not set for other platforms + - name: session_count + type: INTEGER + mode: NULLABLE + description: An optional running counter of the number of sessions for a client. + - name: session_id + type: STRING + mode: NULLABLE + description: An optional UUID uniquely identifying the client's current session. +- name: document_id + type: STRING + mode: NULLABLE + description: The document ID specified in the URI when the client sent this message +- name: events + type: RECORD + mode: REPEATED + fields: + - name: category + type: STRING + mode: NULLABLE + - name: extra + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: STRING + mode: NULLABLE + - name: name + type: STRING + mode: NULLABLE + - name: timestamp + type: INTEGER + mode: NULLABLE +- name: metadata + type: RECORD + mode: NULLABLE + fields: + - name: geo + type: RECORD + mode: NULLABLE + fields: + - name: city + type: STRING + mode: NULLABLE + - name: country + type: STRING + mode: NULLABLE + description: An ISO 3166-1 alpha-2 country code + - name: db_version + type: STRING + mode: NULLABLE + description: The specific geo database version used for this lookup + - name: subdivision1 + type: STRING + mode: NULLABLE + description: First major country subdivision, typically a state, province, or + county + - name: subdivision2 + type: STRING + mode: NULLABLE + description: Second major country subdivision; not applicable for most countries + description: Results of a geographic lookup based on the client's IP address + - name: header + type: RECORD + mode: NULLABLE + fields: + - name: date + type: STRING + mode: NULLABLE + description: Date HTTP header + - name: dnt + type: STRING + mode: NULLABLE + description: DNT (Do Not Track) HTTP header + - name: x_debug_id + type: STRING + mode: NULLABLE + description: X-Debug-Id HTTP header + - name: x_foxsec_ip_reputation + type: STRING + mode: NULLABLE + description: X-Foxsec-IP-Reputation header + - name: x_lb_tags + type: STRING + mode: NULLABLE + description: X-LB-Tags HTTP header + - name: x_pingsender_version + type: STRING + mode: NULLABLE + description: X-PingSender-Version HTTP header + - name: x_source_tags + type: STRING + mode: NULLABLE + description: X-Source-Tags HTTP header + - name: x_telemetry_agent + type: STRING + mode: NULLABLE + description: X-Telemetry-Agent HTTP header + - name: parsed_date + type: TIMESTAMP + mode: NULLABLE + - name: parsed_x_source_tags + type: STRING + mode: REPEATED + - name: parsed_x_lb_tags + type: RECORD + mode: NULLABLE + fields: + - name: tls_version + type: STRING + mode: NULLABLE + - name: tls_cipher_hex + type: STRING + mode: NULLABLE + description: Headers included in the client's HTTP request + - name: isp + type: RECORD + mode: NULLABLE + fields: + - name: db_version + type: STRING + mode: NULLABLE + description: The specific geo ISP database version used for this lookup + - name: name + type: STRING + mode: NULLABLE + description: The name of the ISP associated with the client's IP address + - name: organization + type: STRING + mode: NULLABLE + description: The name of a specific business entity associated with the client's + IP address when available; otherwise the ISP name + description: Results of ISP lookup based on the client's IP address + - name: user_agent + type: RECORD + mode: NULLABLE + fields: + - name: browser + type: STRING + mode: NULLABLE + - name: os + type: STRING + mode: NULLABLE + - name: version + type: STRING + mode: NULLABLE + description: Parsed components of the client's user agent string +- name: metrics + type: RECORD + mode: NULLABLE + fields: + - name: labeled_counter + type: RECORD + mode: NULLABLE + fields: + - name: glean_error_invalid_label + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: INTEGER + mode: NULLABLE + description: 'Counts the number of times a metric was set with an invalid label. + + The labels are the `category.name` identifier of the metric. + + ' + - name: glean_error_invalid_overflow + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: INTEGER + mode: NULLABLE + description: 'Counts the number of times a metric was set a value that overflowed. + + The labels are the `category.name` identifier of the metric. + + ' + - name: glean_error_invalid_state + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: INTEGER + mode: NULLABLE + description: 'Counts the number of times a timing metric was used incorrectly. + + The labels are the `category.name` identifier of the metric. + + ' + - name: glean_error_invalid_value + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: INTEGER + mode: NULLABLE + description: 'Counts the number of times a metric was set to an invalid value. + + The labels are the `category.name` identifier of the metric. + + ' + - name: quantity + type: RECORD + mode: NULLABLE + fields: + - name: top_sites_contile_tile_id + type: INTEGER + mode: NULLABLE + description: 'A unique identifier provided by the AdM for the sponsored TopSites + tile + + ' + - name: string + type: RECORD + mode: NULLABLE + fields: + - name: top_sites_contile_advertiser + type: STRING + mode: NULLABLE + description: 'Advertiser brand for the sponsored TopSites tile + + ' + - name: glean_client_annotation_experimentation_id + type: STRING + mode: NULLABLE + description: 'An experimentation identifier derived and provided by the application + + for the purpose of experimentation enrollment. + + ' + - name: url2 + type: RECORD + mode: NULLABLE + fields: + - name: top_sites_contile_reporting_url + type: STRING + mode: NULLABLE + description: "The AdM reporting endpoint (impression_url for \u201Cimpression\u201D\ + \ event,\nclick_url for \u201Cclick\u201D event).\n" + - name: uuid + type: RECORD + mode: NULLABLE + fields: + - name: top_sites_context_id + type: STRING + mode: NULLABLE + description: 'A UUID that is unjoinable with other browser metrics. This ID + will not be + + shared with AdM, only for internal uses. This ID is shared across all + + contextual services features. + + ' + - name: boolean + type: RECORD + mode: NULLABLE + fields: + - name: customize_home_contile + type: BOOLEAN + mode: NULLABLE + description: 'An indication of whether Contile is enabled to be displayed + + ' + - name: url + type: RECORD + mode: NULLABLE + fields: + - name: top_sites_contile_reporting_url + type: STRING + mode: NULLABLE +- name: normalized_app_name + type: STRING + mode: NULLABLE + description: Set to "Other" if this message contained an unrecognized app name +- name: normalized_channel + type: STRING + mode: NULLABLE + description: Set to "Other" if this message contained an unrecognized channel name +- name: normalized_country_code + type: STRING + mode: NULLABLE + description: An ISO 3166-1 alpha-2 country code +- name: normalized_os + type: STRING + mode: NULLABLE + description: Set to "Other" if this message contained an unrecognized OS name +- name: normalized_os_version + type: STRING + mode: NULLABLE +- name: ping_info + type: RECORD + mode: NULLABLE + fields: + - name: end_time + type: STRING + mode: NULLABLE + - name: experiments + type: RECORD + mode: REPEATED + fields: + - name: key + type: STRING + mode: NULLABLE + - name: value + type: RECORD + mode: NULLABLE + fields: + - name: branch + type: STRING + mode: NULLABLE + - name: extra + type: RECORD + mode: NULLABLE + fields: + - name: type + type: STRING + mode: NULLABLE + - name: enrollment_id + type: STRING + mode: NULLABLE + - name: ping_type + type: STRING + mode: NULLABLE + - name: reason + type: STRING + mode: NULLABLE + - name: seq + type: INTEGER + mode: NULLABLE + - name: start_time + type: STRING + mode: NULLABLE + - name: parsed_start_time + type: TIMESTAMP + mode: NULLABLE + - name: parsed_end_time + type: TIMESTAMP + mode: NULLABLE +- name: sample_id + type: INTEGER + mode: NULLABLE + description: Hashed version of client_id (if present) useful for partitioning; ranges + from 0 to 99 +- name: submission_timestamp + type: TIMESTAMP + mode: NULLABLE + description: Time when the ingestion edge server accepted this message diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:34:20.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:20:19.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:34:21.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:20:20.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:34:20.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:20:19.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:34:20.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:20:19.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:34:21.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:20:19.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:34:20.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:20:20.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-06 19:34:21.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-06 19:19:28.000000000 +0000 @@ -1,7 +1,9 @@ fields: - name: postal_district_code type: STRING - mode: NULLABLE + mode: REQUIRED + description: One-character Canadian postal district code. - name: province_code type: STRING mode: NULLABLE + description: Two-character Canadian province/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-06 19:34:21.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-06 19:19:28.000000000 +0000 @@ -1,28 +1,47 @@ fields: - name: name + description: Official country name per ISO 3166 type: STRING - mode: NULLABLE + mode: REQUIRED - name: code + description: ISO 3166 alpha-2 country code type: STRING - mode: NULLABLE + mode: REQUIRED - name: code_3 + description: ISO 3166 alpha-3 country code type: STRING - mode: NULLABLE + mode: REQUIRED - name: region_name + description: Region name. These are based on the UN Statistics Division standard + country or area codes for statistical use (M49), but with the "Americas" region + split into "North America" and "South America". type: STRING - mode: NULLABLE + mode: REQUIRED - name: subregion_name + description: Sub-region name. These are based on UN Statistics Division standard + country or area codes for statistical use (M49), but with the "Latin America and the + Caribbean" and "Sub-Saharan Africa" sub-regions split into more specific + sub-regions. type: STRING - mode: NULLABLE + mode: REQUIRED - name: pocket_available_on_newtab - type: BOOLEAN - mode: NULLABLE + description: Whether Pocket is available on the newtab page in this country. Note + that Pocket might only be available in certain locales/languages within a country. + type: BOOL + mode: REQUIRED - name: mozilla_vpn_available - type: BOOLEAN - mode: NULLABLE + description: Whether Mozilla VPN is available in this country. + type: BOOL + mode: REQUIRED - name: sponsored_tiles_available_on_newtab - type: BOOLEAN - mode: NULLABLE + description: Whether sponsored tiles are available on the newtab page in this country. + Note that Pocket might only be available in certain locales/languages within a + country. + type: BOOL + mode: REQUIRED - name: ads_value_tier + description: Lowercase label detailing the monetary value tier that Mozilla Ads + assign to that region based on market size and our existing products, e.g., tier + 1, tier 2, etc. type: STRING - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-06 19:34:21.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-06 19:19:28.000000000 +0000 @@ -1,7 +1,10 @@ fields: - name: name + description: An alias for a country's name (including misspellings and alternate + encodings). type: STRING - mode: NULLABLE + mode: REQUIRED - name: code + description: ISO 3166 alpha-2 country code type: STRING - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-06 19:34:21.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-06 19:19:28.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: incident +- mode: NULLABLE + name: incident type: STRING - mode: NULLABLE -- name: description +- mode: NULLABLE + name: description type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE -- name: product +- mode: NULLABLE + name: product type: STRING - mode: NULLABLE -- name: version +- mode: NULLABLE + name: version type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-06 19:34:21.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-06 19:19:28.000000000 +0000 @@ -1,16 +1,27 @@ fields: -- name: value +- mode: NULLABLE + description: Hex value assigned to the TLS cipher, in format like "0x00,0x84"; note + some values are ranges or contain wildcards + name: value type: STRING - mode: NULLABLE -- name: description +- mode: NULLABLE + description: Human-readable nam ```

⚠️ Only part of the diff is displayed.

Link to full diff

dataops-ci-bot commented 4 weeks ago

Integration report for "switch to string"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 19:33:20.000000000 +0000 @@ -0,0 +1,82 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_google_analytics_derived_ga4", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_delta=datetime.timedelta(seconds=18000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table='ga_desktop_conversions_v1${{ macros.ds_format(macros.ds_add(ds, -2), "%Y-%m-%d", "%Y%m%d") }}', + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + parameters=["activity_date:DATE:{{macros.ds_add(ds, -2)}}"] + + ["submission_date:DATE:{{ds}}"], + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 19:47:16.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 19:33:20.000000000 +0000 @@ -153,6 +153,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v2_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v2_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=68400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v2_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) + checks__warn_ga_derived__blogs_goals__v2 = bigquery_dq_check( task_id="checks__warn_ga_derived__blogs_goals__v2", source_table="blogs_goals_v2", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:29:31.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.events_v1` + `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.events_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- name: submission_date +- mode: NULLABLE + name: submission_date type: DATE - mode: NULLABLE -- name: source +- mode: NULLABLE + name: source type: STRING - mode: NULLABLE -- name: event_type +- mode: NULLABLE + name: event_type type: STRING - mode: NULLABLE -- name: form_factor +- mode: NULLABLE + name: form_factor type: STRING - mode: NULLABLE -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: subdivision1 +- mode: NULLABLE + name: subdivision1 type: STRING - mode: NULLABLE -- name: advertiser +- mode: NULLABLE + name: advertiser type: STRING - mode: NULLABLE -- name: release_channel +- mode: NULLABLE + name: release_channel type: STRING - mode: NULLABLE -- name: position +- mode: NULLABLE + name: position type: INTEGER - mode: NULLABLE -- name: provider +- mode: NULLABLE + name: provider type: STRING - mode: NULLABLE -- name: match_type +- mode: NULLABLE + name: match_type type: STRING - mode: NULLABLE -- name: normalized_os +- mode: NULLABLE + name: normalized_os type: STRING - mode: NULLABLE -- name: suggest_data_sharing_enabled +- mode: NULLABLE + name: suggest_data_sharing_enabled type: BOOLEAN - mode: NULLABLE -- name: event_count +- mode: NULLABLE + name: event_count type: INTEGER - mode: NULLABLE -- name: user_count +- mode: NULLABLE + name: user_count type: INTEGER - mode: NULLABLE -- name: query_type +- mode: NULLABLE + name: query_type type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- name: submission_date +- mode: NULLABLE + name: submission_date type: DATE - mode: NULLABLE -- name: form_factor +- mode: NULLABLE + name: form_factor type: STRING - mode: NULLABLE -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: advertiser +- mode: NULLABLE + name: advertiser type: STRING - mode: NULLABLE -- name: normalized_os +- mode: NULLABLE + name: normalized_os type: STRING - mode: NULLABLE -- name: release_channel +- mode: NULLABLE + name: release_channel type: STRING - mode: NULLABLE -- name: position +- mode: NULLABLE + name: position type: INTEGER - mode: NULLABLE -- name: provider +- mode: NULLABLE + name: provider type: STRING - mode: NULLABLE -- name: match_type +- mode: NULLABLE + name: match_type type: STRING - mode: NULLABLE -- name: suggest_data_sharing_enabled +- mode: NULLABLE + name: suggest_data_sharing_enabled type: BOOLEAN - mode: NULLABLE -- name: impression_count +- mode: NULLABLE + name: impression_count type: INTEGER - mode: NULLABLE -- name: click_count +- mode: NULLABLE + name: click_count type: INTEGER - mode: NULLABLE -- name: query_type +- mode: NULLABLE + name: query_type type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -26,9 +26,6 @@ - name: adjust_network type: STRING mode: NULLABLE -- name: install_source - type: STRING - mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -48,10 +48,6 @@ description: 'The type of source of a client installation. ' -- name: install_source - type: STRING - mode: NULLABLE - description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:29:31.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/query.sql 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/query.sql 2024-06-06 19:29:31.000000000 +0000 @@ -69,13 +69,10 @@ COALESCE(metrics.boolean.urlbar_pref_suggest_nonsponsored, FALSE) AS pref_fx_suggestions, mozfun.map.get_key(extra, "engagement_type") AS engagement_type, mozfun.map.get_key(extra, "interaction") AS interaction, - SAFE_CAST(mozfun.map.get_key(extra, "n_chars") AS int) AS num_chars_typed, - SAFE_CAST(mozfun.map.get_key(extra, "n_results") AS int) AS num_total_results, + CAST(mozfun.map.get_key(extra, "n_chars") AS int) AS num_chars_typed, + CAST(mozfun.map.get_key(extra, "n_results") AS int) AS num_total_results, --If 0, then no result was selected. - NULLIF( - SAFE_CAST(mozfun.map.get_key(extra, "selected_position") AS int), - 0 - ) AS selected_position, + NULLIF(CAST(mozfun.map.get_key(extra, "selected_position") AS int), 0) AS selected_position, mozfun.map.get_key(extra, "selected_result") AS selected_result, enumerated_array( SPLIT(mozfun.map.get_key(extra, "results"), ','), diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: city +- mode: NULLABLE + name: city type: STRING - mode: NULLABLE -- name: datetime +- mode: NULLABLE + name: datetime type: TIMESTAMP - mode: NULLABLE -- name: proportion_undefined +- mode: NULLABLE + name: proportion_undefined type: FLOAT - mode: NULLABLE -- name: proportion_timeout +- mode: NULLABLE + name: proportion_timeout type: FLOAT - mode: NULLABLE -- name: proportion_abort +- mode: NULLABLE + name: proportion_abort type: FLOAT - mode: NULLABLE -- name: proportion_unreachable +- mode: NULLABLE + name: proportion_unreachable type: FLOAT - mode: NULLABLE -- name: proportion_terminated +- mode: NULLABLE + name: proportion_terminated type: FLOAT - mode: NULLABLE -- name: proportion_channel_open +- mode: NULLABLE + name: proportion_channel_open type: FLOAT - mode: NULLABLE -- name: avg_dns_success_time +- mode: NULLABLE + name: avg_dns_success_time type: FLOAT - mode: NULLABLE -- name: missing_dns_success +- mode: NULLABLE + name: missing_dns_success type: FLOAT - mode: NULLABLE -- name: avg_dns_failure_time +- mode: NULLABLE + name: avg_dns_failure_time type: FLOAT - mode: NULLABLE -- name: missing_dns_failure +- mode: NULLABLE + name: missing_dns_failure type: FLOAT - mode: NULLABLE -- name: count_dns_failure +- mode: NULLABLE + name: count_dns_failure type: FLOAT - mode: NULLABLE -- name: ssl_error_prop +- mode: NULLABLE + name: ssl_error_prop type: FLOAT - mode: NULLABLE -- name: avg_tls_handshake_time +- mode: NULLABLE + name: avg_tls_handshake_time type: FLOAT - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql 2024-06-06 19:28:45.000000000 +0000 @@ -42,34 +42,20 @@ jobs AS ( -- https://cloud.google.com/bigquery/docs/information-schema-jobs SELECT - creation_time, - start_time, - end_time, - state, - error_result, - job_id, - project_id, - total_bytes_processed AS bytes_complete, - total_slot_ms AS slot_ms, + * FROM `moz-fx-data-shredder.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT` UNION ALL SELECT - creation_time, - start_time, - end_time, - state, - error_result, - job_id, - project_id, - total_bytes_processed AS bytes_complete, - total_slot_ms AS slot_ms, + * FROM `moz-fx-data-bq-batch-prod.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT` ), successful_jobs AS ( SELECT - * + *, + total_bytes_processed AS bytes_complete, + total_slot_ms AS slot_ms, FROM jobs WHERE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-06 19:29:31.000000000 +0000 @@ -45,7 +45,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -572,7 +572,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -582,7 +582,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -663,7 +663,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -673,7 +673,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -754,7 +754,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -764,7 +764,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -1990,7 +1990,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.events_v1` + `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2000,7 +2000,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.events_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 19:30:32.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 19:28:45.000000000 +0000 @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 19:30:30.000000000 +0000 @@ -0,0 +1,30 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed + date_partition_parameter: activity_date + date_partition_offset: -2 + parameters: + - submission_date:DATE:{{ds}} +bigquery: + time_partitioning: + type: day + field: activity_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 19:28:45.000000000 +0000 @@ -0,0 +1,29 @@ +SELECT + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_date, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.gclid_conversions_v2` UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date = DATE_SUB(@activity_date, INTERVAL 2 DAY) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -0,0 +1,13 @@ +fields: +- name: activity_date + mode: NULLABLE + type: STRING + description: Activity Date +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -1,13 +1,7 @@ fields: -- name: build_hour - type: DATETIME - mode: NULLABLE - description: null -- name: geckoview_major_version - type: INTEGER - mode: NULLABLE - description: null -- name: n_pings - type: INTEGER - mode: NULLABLE - description: null +- type: DATETIME + name: build_hour +- type: INTEGER + name: geckoview_major_version +- type: INTEGER + name: n_pings diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:29:32.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:29:31.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:29:31.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -1,7 +1,9 @@ fields: - name: postal_district_code type: STRING - mode: NULLABLE + mode: REQUIRED + description: One-character Canadian postal district code. - name: province_code type: STRING mode: NULLABLE + description: Two-character Canadian province/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -1,28 +1,47 @@ fields: - name: name + description: Official country name per ISO 3166 type: STRING - mode: NULLABLE + mode: REQUIRED - name: code + description: ISO 3166 alpha-2 country code type: STRING - mode: NULLABLE + mode: REQUIRED - name: code_3 + description: ISO 3166 alpha-3 country code type: STRING - mode: NULLABLE + mode: REQUIRED - name: region_name + description: Region name. These are based on the UN Statistics Division standard + country or area codes for statistical use (M49), but with the "Americas" region + split into "North America" and "South America". type: STRING - mode: NULLABLE + mode: REQUIRED - name: subregion_name + description: Sub-region name. These are based on UN Statistics Division standard + country or area codes for statistical use (M49), but with the "Latin America and the + Caribbean" and "Sub-Saharan Africa" sub-regions split into more specific + sub-regions. type: STRING - mode: NULLABLE + mode: REQUIRED - name: pocket_available_on_newtab - type: BOOLEAN - mode: NULLABLE + description: Whether Pocket is available on the newtab page in this country. Note + that Pocket might only be available in certain locales/languages within a country. + type: BOOL + mode: REQUIRED - name: mozilla_vpn_available - type: BOOLEAN - mode: NULLABLE + description: Whether Mozilla VPN is available in this country. + type: BOOL + mode: REQUIRED - name: sponsored_tiles_available_on_newtab - type: BOOLEAN - mode: NULLABLE + description: Whether sponsored tiles are available on the newtab page in this country. + Note that Pocket might only be available in certain locales/languages within a + country. + type: BOOL + mode: REQUIRED - name: ads_value_tier + description: Lowercase label detailing the monetary value tier that Mozilla Ads + assign to that region based on market size and our existing products, e.g., tier + 1, tier 2, etc. type: STRING - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -1,7 +1,10 @@ fields: - name: name + description: An alias for a country's name (including misspellings and alternate + encodings). type: STRING - mode: NULLABLE + mode: REQUIRED - name: code + description: ISO 3166 alpha-2 country code type: STRING - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: incident +- mode: NULLABLE + name: incident type: STRING - mode: NULLABLE -- name: description +- mode: NULLABLE + name: description type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE -- name: product +- mode: NULLABLE + name: product type: STRING - mode: NULLABLE -- name: version +- mode: NULLABLE + name: version type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -1,16 +1,27 @@ fields: -- name: value +- mode: NULLABLE + description: Hex value assigned to the TLS cipher, in format like "0x00,0x84"; note + some values are ranges or contain wildcards + name: value type: STRING - mode: NULLABLE -- name: description +- mode: NULLABLE + description: Human-readable name of the TLS cipher + name: description type: STRING - mode: NULLABLE -- name: dtls_ok +- mode: NULLABLE + description: Any TLS cipher suite that is specified for use with DTLS MUST define + limits on the use of the associated AEAD function that preserves margins for both + confidentiality and integrity, as specified in [RFC-ietf-tls-dtls13-43] + name: dtls_ok type: BOOLEAN - mode: NULLABLE -- name: recommended +- mode: NULLABLE + description: Whether the TLS cipher is recommended by the IETF. If an item is not + marked as "recommended", it does not necessarily mean that it is flawed; rather, + it indicates that the item either has not been through the IETF consensus process, + has limited applicability, or is intended only for specific use cases + name: recommended type: BOOLEAN - mode: NULLABLE -- name: reference +- mode: NULLABLE + description: RFCs or associated reference material for the TLS cipher + name: reference type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -1,13 +1,17 @@ fields: - name: code_3 + description: ISO 639 alpha-3 language code. type: STRING - mode: NULLABLE + mode: REQUIRED - name: code_2 + description: ISO 639 alpha-2 language code (if any). type: STRING mode: NULLABLE - name: name + description: Language name. type: STRING - mode: NULLABLE + mode: REQUIRED - name: other_names + description: Other names for the language (if any). type: STRING mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -1,19 +1,19 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -1,25 +1,25 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: document_version +- mode: NULLABLE + name: document_version type: STRING - mode: NULLABLE -- name: path +- mode: NULLABLE + name: path type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: document_version +- mode: NULLABLE + name: document_version type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: path +- mode: NULLABLE + name: path type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -1,7 +1,7 @@ fields: - name: score type: NUMERIC - mode: NULLABLE + mode: REQUIRED - name: value type: NUMERIC - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-06 19:44:19.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-06 19:28:45.000000000 +0000 @@ -1,7 +1,9 @@ fields: - name: zip_code_prefix type: STRING - mode: NULLABLE + mode: REQUIRED + description: Three-digit US ZIP code prefix. - name: state_code type: STRING mode: NULLABLE + description: Two-character US state/territory code (if any). ```

Link to full diff

dataops-ci-bot commented 4 weeks ago

Integration report for "update schema and query"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 19:53:45.000000000 +0000 @@ -0,0 +1,82 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_google_analytics_derived_ga4", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_delta=datetime.timedelta(seconds=18000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table='ga_desktop_conversions_v1${{ macros.ds_format(macros.ds_add(ds, -2), "%Y-%m-%d", "%Y%m%d") }}', + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + parameters=["activity_date:DATE:{{macros.ds_add(ds, -2)}}"] + + ["submission_date:DATE:{{ds}}"], + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 19:39:15.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 19:53:45.000000000 +0000 @@ -153,6 +153,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v2_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v2_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=68400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v2_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) + checks__warn_ga_derived__blogs_goals__v2 = bigquery_dq_check( task_id="checks__warn_ga_derived__blogs_goals__v2", source_table="blogs_goals_v2", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:36:11.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:38:01.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.events_v1` + `moz-fx-data-shared-prod.bedrock_live.interaction_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-06 19:44:01.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: source + mode: NULLABLE +- name: source type: STRING -- mode: NULLABLE - name: event_type + mode: NULLABLE +- name: event_type type: STRING -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: subdivision1 + mode: NULLABLE +- name: subdivision1 type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: event_count + mode: NULLABLE +- name: event_count type: INTEGER -- mode: NULLABLE - name: user_count + mode: NULLABLE +- name: user_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-06 19:44:01.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- mode: NULLABLE - name: submission_date +- name: submission_date type: DATE -- mode: NULLABLE - name: form_factor + mode: NULLABLE +- name: form_factor type: STRING -- mode: NULLABLE - name: country + mode: NULLABLE +- name: country type: STRING -- mode: NULLABLE - name: advertiser + mode: NULLABLE +- name: advertiser type: STRING -- mode: NULLABLE - name: normalized_os + mode: NULLABLE +- name: normalized_os type: STRING -- mode: NULLABLE - name: release_channel + mode: NULLABLE +- name: release_channel type: STRING -- mode: NULLABLE - name: position + mode: NULLABLE +- name: position type: INTEGER -- mode: NULLABLE - name: provider + mode: NULLABLE +- name: provider type: STRING -- mode: NULLABLE - name: match_type + mode: NULLABLE +- name: match_type type: STRING -- mode: NULLABLE - name: suggest_data_sharing_enabled + mode: NULLABLE +- name: suggest_data_sharing_enabled type: BOOLEAN -- mode: NULLABLE - name: impression_count + mode: NULLABLE +- name: impression_count type: INTEGER -- mode: NULLABLE - name: click_count + mode: NULLABLE +- name: click_count type: INTEGER -- mode: NULLABLE - name: query_type + mode: NULLABLE +- name: query_type type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-06 19:47:45.000000000 +0000 @@ -26,6 +26,9 @@ - name: adjust_network type: STRING mode: NULLABLE +- name: install_source + type: STRING + mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-06 19:47:45.000000000 +0000 @@ -48,6 +48,10 @@ description: 'The type of source of a client installation. ' +- name: install_source + type: STRING + mode: NULLABLE + description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:36:10.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:38:01.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:36:08.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:38:01.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -80,7 +80,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/query.sql 2024-06-06 19:36:08.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/urlbar_events_v2/query.sql 2024-06-06 19:36:22.000000000 +0000 @@ -69,13 +69,10 @@ COALESCE(metrics.boolean.urlbar_pref_suggest_nonsponsored, FALSE) AS pref_fx_suggestions, mozfun.map.get_key(extra, "engagement_type") AS engagement_type, mozfun.map.get_key(extra, "interaction") AS interaction, - SAFE_CAST(mozfun.map.get_key(extra, "n_chars") AS int) AS num_chars_typed, - SAFE_CAST(mozfun.map.get_key(extra, "n_results") AS int) AS num_total_results, + CAST(mozfun.map.get_key(extra, "n_chars") AS int) AS num_chars_typed, + CAST(mozfun.map.get_key(extra, "n_results") AS int) AS num_total_results, --If 0, then no result was selected. - NULLIF( - SAFE_CAST(mozfun.map.get_key(extra, "selected_position") AS int), - 0 - ) AS selected_position, + NULLIF(CAST(mozfun.map.get_key(extra, "selected_position") AS int), 0) AS selected_position, mozfun.map.get_key(extra, "selected_result") AS selected_result, enumerated_array( SPLIT(mozfun.map.get_key(extra, "results"), ','), diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-06 19:44:34.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- mode: NULLABLE - name: country +- name: country type: STRING -- mode: NULLABLE - name: city + mode: NULLABLE +- name: city type: STRING -- mode: NULLABLE - name: datetime + mode: NULLABLE +- name: datetime type: TIMESTAMP -- mode: NULLABLE - name: proportion_undefined + mode: NULLABLE +- name: proportion_undefined type: FLOAT -- mode: NULLABLE - name: proportion_timeout + mode: NULLABLE +- name: proportion_timeout type: FLOAT -- mode: NULLABLE - name: proportion_abort + mode: NULLABLE +- name: proportion_abort type: FLOAT -- mode: NULLABLE - name: proportion_unreachable + mode: NULLABLE +- name: proportion_unreachable type: FLOAT -- mode: NULLABLE - name: proportion_terminated + mode: NULLABLE +- name: proportion_terminated type: FLOAT -- mode: NULLABLE - name: proportion_channel_open + mode: NULLABLE +- name: proportion_channel_open type: FLOAT -- mode: NULLABLE - name: avg_dns_success_time + mode: NULLABLE +- name: avg_dns_success_time type: FLOAT -- mode: NULLABLE - name: missing_dns_success + mode: NULLABLE +- name: missing_dns_success type: FLOAT -- mode: NULLABLE - name: avg_dns_failure_time + mode: NULLABLE +- name: avg_dns_failure_time type: FLOAT -- mode: NULLABLE - name: missing_dns_failure + mode: NULLABLE +- name: missing_dns_failure type: FLOAT -- mode: NULLABLE - name: count_dns_failure + mode: NULLABLE +- name: count_dns_failure type: FLOAT -- mode: NULLABLE - name: ssl_error_prop + mode: NULLABLE +- name: ssl_error_prop type: FLOAT -- mode: NULLABLE - name: avg_tls_handshake_time + mode: NULLABLE +- name: avg_tls_handshake_time type: FLOAT + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql 2024-06-06 19:35:33.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring/shredder_progress/view.sql 2024-06-06 19:36:16.000000000 +0000 @@ -42,34 +42,20 @@ jobs AS ( -- https://cloud.google.com/bigquery/docs/information-schema-jobs SELECT - creation_time, - start_time, - end_time, - state, - error_result, - job_id, - project_id, - total_bytes_processed AS bytes_complete, - total_slot_ms AS slot_ms, + * FROM `moz-fx-data-shredder.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT` UNION ALL SELECT - creation_time, - start_time, - end_time, - state, - error_result, - job_id, - project_id, - total_bytes_processed AS bytes_complete, - total_slot_ms AS slot_ms, + * FROM `moz-fx-data-bq-batch-prod.region-us.INFORMATION_SCHEMA.JOBS_BY_PROJECT` ), successful_jobs AS ( SELECT - * + *, + total_bytes_processed AS bytes_complete, + total_slot_ms AS slot_ms, FROM jobs WHERE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-06 19:36:09.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-06 19:39:40.000000000 +0000 @@ -45,7 +45,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -75,7 +75,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -572,7 +572,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -582,7 +582,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -592,7 +592,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -663,7 +663,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -673,7 +673,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -683,7 +683,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -754,7 +754,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -764,7 +764,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -774,7 +774,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1555,7 +1555,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1575,7 +1575,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1646,7 +1646,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1666,7 +1666,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1737,7 +1737,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1757,7 +1757,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1828,7 +1828,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1848,7 +1848,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1990,7 +1990,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -2010,7 +2010,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.events_v1` + `moz-fx-data-shared-prod.bedrock_stable.interaction_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2162,7 +2162,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -2172,7 +2172,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 19:50:48.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 19:36:16.000000000 +0000 @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 19:50:48.000000000 +0000 @@ -0,0 +1,30 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed + date_partition_parameter: activity_date + date_partition_offset: -2 + parameters: + - submission_date:DATE:{{ds}} +bigquery: + time_partitioning: + type: day + field: activity_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 19:36:16.000000000 +0000 @@ -0,0 +1,30 @@ +SELECT + activity_date AS activity_date, + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_datetime, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.gclid_conversions_v2` UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date = DATE_SUB(@activity_date, INTERVAL 2 DAY) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 19:36:16.000000000 +0000 @@ -0,0 +1,17 @@ +fields: +- name: activity_date + mode: NULLABLE + type: DATE + description: Activity Date +- name: activity_datetime + mode: NULLABLE + type: STRING + description: Activity Datetime +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:36:09.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:38:01.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.main_v1` + `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-06 19:44:12.000000000 +0000 @@ -1,7 +1,13 @@ fields: -- type: DATETIME - name: build_hour -- type: INTEGER - name: geckoview_major_version -- type: INTEGER - name: n_pings +- name: build_hour + type: DATETIME + mode: NULLABLE + description: null +- name: geckoview_major_version + type: INTEGER + mode: NULLABLE + description: null +- name: n_pings + type: INTEGER + mode: NULLABLE + description: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:36:10.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:38:03.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:36:08.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:38:03.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:36:10.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:38:04.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:36:11.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:38:03.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:36:10.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:38:04.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:36:08.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 19:38:04.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-06 19:45:03.000000000 +0000 @@ -1,9 +1,7 @@ fields: - name: postal_district_code type: STRING - mode: REQUIRED - description: One-character Canadian postal district code. + mode: NULLABLE - name: province_code type: STRING mode: NULLABLE - description: Two-character Canadian province/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-06 19:45:03.000000000 +0000 @@ -1,47 +1,28 @@ fields: - name: name - description: Official country name per ISO 3166 type: STRING - mode: REQUIRED + mode: NULLABLE - name: code - description: ISO 3166 alpha-2 country code type: STRING - mode: REQUIRED + mode: NULLABLE - name: code_3 - description: ISO 3166 alpha-3 country code type: STRING - mode: REQUIRED + mode: NULLABLE - name: region_name - description: Region name. These are based on the UN Statistics Division standard - country or area codes for statistical use (M49), but with the "Americas" region - split into "North America" and "South America". type: STRING - mode: REQUIRED + mode: NULLABLE - name: subregion_name - description: Sub-region name. These are based on UN Statistics Division standard - country or area codes for statistical use (M49), but with the "Latin America and the - Caribbean" and "Sub-Saharan Africa" sub-regions split into more specific - sub-regions. type: STRING - mode: REQUIRED + mode: NULLABLE - name: pocket_available_on_newtab - description: Whether Pocket is available on the newtab page in this country. Note - that Pocket might only be available in certain locales/languages within a country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: mozilla_vpn_available - description: Whether Mozilla VPN is available in this country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: sponsored_tiles_available_on_newtab - description: Whether sponsored tiles are available on the newtab page in this country. - Note that Pocket might only be available in certain locales/languages within a - country. - type: BOOL - mode: REQUIRED + type: BOOLEAN + mode: NULLABLE - name: ads_value_tier - description: Lowercase label detailing the monetary value tier that Mozilla Ads - assign to that region based on market size and our existing products, e.g., tier - 1, tier 2, etc. type: STRING - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-06 19:45:03.000000000 +0000 @@ -1,10 +1,7 @@ fields: - name: name - description: An alias for a country's name (including misspellings and alternate - encodings). type: STRING - mode: REQUIRED + mode: NULLABLE - name: code - description: ISO 3166 alpha-2 country code type: STRING - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-06 19:45:04.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: incident + mode: NULLABLE +- name: incident type: STRING -- mode: NULLABLE - name: description + mode: NULLABLE +- name: description type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING -- mode: NULLABLE - name: product + mode: NULLABLE +- name: product type: STRING -- mode: NULLABLE - name: version + mode: NULLABLE +- name: version type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-06 19:45:02.000000000 +0000 @@ -1,27 +1,16 @@ fields: -- mode: NULLABLE - description: Hex value assigned to the TLS cipher, in format like "0x00,0x84"; note - some values are ranges or contain wildcards - name: value +- name: value type: STRING -- mode: NULLABLE - description: Human-readable name of the TLS cipher - name: description + mode: NULLABLE +- name: description type: STRING -- mode: NULLABLE - description: Any TLS cipher suite that is specified for use with DTLS MUST define - limits on the use of the associated AEAD function that preserves margins for both - confidentiality and integrity, as specified in [RFC-ietf-tls-dtls13-43] - name: dtls_ok + mode: NULLABLE +- name: dtls_ok type: BOOLEAN -- mode: NULLABLE - description: Whether the TLS cipher is recommended by the IETF. If an item is not - marked as "recommended", it does not necessarily mean that it is flawed; rather, - it indicates that the item either has not been through the IETF consensus process, - has limited applicability, or is intended only for specific use cases - name: recommended + mode: NULLABLE +- name: recommended type: BOOLEAN -- mode: NULLABLE - description: RFCs or associated reference material for the TLS cipher - name: reference + mode: NULLABLE +- name: reference type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-06 19:45:03.000000000 +0000 @@ -1,17 +1,13 @@ fields: - name: code_3 - description: ISO 639 alpha-3 language code. type: STRING - mode: REQUIRED + mode: NULLABLE - name: code_2 - description: ISO 639 alpha-2 language code (if any). type: STRING mode: NULLABLE - name: name - description: Language name. type: STRING - mode: REQUIRED + mode: NULLABLE - name: other_names - description: Other names for the language (if any). type: STRING mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-06 19:45:03.000000000 +0000 @@ -1,19 +1,19 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-06 19:45:04.000000000 +0000 @@ -1,25 +1,25 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: document_version + mode: NULLABLE +- name: document_version type: STRING -- mode: NULLABLE - name: path + mode: NULLABLE +- name: path type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-06 19:45:03.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: document_version + mode: NULLABLE +- name: document_version type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-06 19:45:03.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- mode: NULLABLE - name: start_date +- name: start_date type: DATE -- mode: NULLABLE - name: end_date + mode: NULLABLE +- name: end_date type: DATE -- mode: NULLABLE - name: document_namespace + mode: NULLABLE +- name: document_namespace type: STRING -- mode: NULLABLE - name: document_type + mode: NULLABLE +- name: document_type type: STRING -- mode: NULLABLE - name: path + mode: NULLABLE +- name: path type: STRING -- mode: NULLABLE - name: notes + mode: NULLABLE +- name: notes type: STRING -- mode: NULLABLE - name: bug + mode: NULLABLE +- name: bug type: STRING + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-06 19:45:03.000000000 +0000 @@ -1,7 +1,7 @@ fields: - name: score type: NUMERIC - mode: REQUIRED + mode: NULLABLE - name: value type: NUMERIC - mode: REQUIRED + mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-06 19:35:32.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-06 19:45:03.000000000 +0000 @@ -1,9 +1,7 @@ fields: - name: zip_code_prefix type: STRING - mode: REQUIRED - description: Three-digit US ZIP code prefix. + mode: NULLABLE - name: state_code type: STRING mode: NULLABLE - description: Two-character US state/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml 2024-06-06 19:36:09.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml 2024-06-06 19:45:33.000000000 +0000 @@ -2,21 +2,18 @@ - name: date type: DATE mode: NULLABLE - description: null - name: product type: STRING mode: NULLABLE - name: category type: STRING mode: NULLABLE - description: null - name: channel type: STRING mode: NULLABLE - name: build_number type: INTEGER mode: NULLABLE - description: null - name: release_date type: DATE mode: NULLABLE ```

Link to full diff

dataops-ci-bot commented 4 weeks ago

Integration report for "Merge branch 'main' into census_desktop_conv_evnt_feed"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 20:28:02.000000000 +0000 @@ -0,0 +1,82 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_google_analytics_derived_ga4", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_delta=datetime.timedelta(seconds=18000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table='ga_desktop_conversions_v1${{ macros.ds_format(macros.ds_add(ds, -2), "%Y-%m-%d", "%Y%m%d") }}', + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + parameters=["activity_date:DATE:{{macros.ds_add(ds, -2)}}"] + + ["submission_date:DATE:{{ds}}"], + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 20:27:01.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 20:28:02.000000000 +0000 @@ -153,6 +153,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v2_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v2_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=68400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v2_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) + checks__warn_ga_derived__blogs_goals__v2 = bigquery_dq_check( task_id="checks__warn_ga_derived__blogs_goals__v2", source_table="blogs_goals_v2", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 20:25:20.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 20:23:28.000000000 +0000 @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 20:25:18.000000000 +0000 @@ -0,0 +1,30 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed + date_partition_parameter: activity_date + date_partition_offset: -2 + parameters: + - submission_date:DATE:{{ds}} +bigquery: + time_partitioning: + type: day + field: activity_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 20:23:28.000000000 +0000 @@ -0,0 +1,30 @@ +SELECT + activity_date AS activity_date, + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_datetime, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.gclid_conversions_v2` UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date = DATE_SUB(@activity_date, INTERVAL 2 DAY) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 20:23:28.000000000 +0000 @@ -0,0 +1,17 @@ +fields: +- name: activity_date + mode: NULLABLE + type: DATE + description: Activity Date +- name: activity_datetime + mode: NULLABLE + type: STRING + description: Activity Datetime +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name ```

Link to full diff

dataops-ci-bot commented 4 weeks ago

Integration report for "Update query.sql"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 20:29:44.000000000 +0000 @@ -0,0 +1,82 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_google_analytics_derived_ga4", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_delta=datetime.timedelta(seconds=18000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table='ga_desktop_conversions_v1${{ macros.ds_format(macros.ds_add(ds, -2), "%Y-%m-%d", "%Y%m%d") }}', + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + parameters=["activity_date:DATE:{{macros.ds_add(ds, -2)}}"] + + ["submission_date:DATE:{{ds}}"], + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 20:28:27.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 20:29:44.000000000 +0000 @@ -153,6 +153,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v2_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v2_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=68400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v2_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) + checks__warn_ga_derived__blogs_goals__v2 = bigquery_dq_check( task_id="checks__warn_ga_derived__blogs_goals__v2", source_table="blogs_goals_v2", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 20:26:30.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 20:24:41.000000000 +0000 @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 20:26:28.000000000 +0000 @@ -0,0 +1,30 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed + date_partition_parameter: activity_date + date_partition_offset: -2 + parameters: + - submission_date:DATE:{{ds}} +bigquery: + time_partitioning: + type: day + field: activity_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 20:24:41.000000000 +0000 @@ -0,0 +1,30 @@ +SELECT + activity_date, + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_datetime, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.gclid_conversions_v2` UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date = DATE_SUB(@activity_date, INTERVAL 2 DAY) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 20:24:41.000000000 +0000 @@ -0,0 +1,17 @@ +fields: +- name: activity_date + mode: NULLABLE + type: DATE + description: Activity Date +- name: activity_datetime + mode: NULLABLE + type: STRING + description: Activity Datetime +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name ```

Link to full diff

dataops-ci-bot commented 4 weeks ago

Integration report for "Update view.sql"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 20:33:19.000000000 +0000 @@ -0,0 +1,82 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_google_analytics_derived_ga4", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_delta=datetime.timedelta(seconds=18000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table='ga_desktop_conversions_v1${{ macros.ds_format(macros.ds_add(ds, -2), "%Y-%m-%d", "%Y%m%d") }}', + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + parameters=["activity_date:DATE:{{macros.ds_add(ds, -2)}}"] + + ["submission_date:DATE:{{ds}}"], + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 20:31:58.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 20:33:19.000000000 +0000 @@ -153,6 +153,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v2_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v2_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=68400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v2_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) + checks__warn_ga_derived__blogs_goals__v2 = bigquery_dq_check( task_id="checks__warn_ga_derived__blogs_goals__v2", source_table="blogs_goals_v2", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 20:30:30.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 20:28:46.000000000 +0000 @@ -0,0 +1,10 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + activity_date AS activity_dt, + activity_datetime AS activity_date, + gclid, + conversion_name +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 20:30:30.000000000 +0000 @@ -0,0 +1,30 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed + date_partition_parameter: activity_date + date_partition_offset: -2 + parameters: + - submission_date:DATE:{{ds}} +bigquery: + time_partitioning: + type: day + field: activity_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 20:28:46.000000000 +0000 @@ -0,0 +1,30 @@ +SELECT + activity_date, + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_datetime, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.gclid_conversions_v2` UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date = DATE_SUB(@activity_date, INTERVAL 2 DAY) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 20:28:46.000000000 +0000 @@ -0,0 +1,17 @@ +fields: +- name: activity_date + mode: NULLABLE + type: DATE + description: Activity Date +- name: activity_datetime + mode: NULLABLE + type: STRING + description: Activity Datetime +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name ```

Link to full diff

dataops-ci-bot commented 4 weeks ago

Integration report for "Add a run date column and use that to partition"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 21:01:02.000000000 +0000 @@ -0,0 +1,82 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_google_analytics_derived_ga4", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_delta=datetime.timedelta(seconds=18000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table='ga_desktop_conversions_v1${{ macros.ds_format(macros.ds_add(ds, -2), "%Y-%m-%d", "%Y%m%d") }}', + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + parameters=["activity_date:DATE:{{macros.ds_add(ds, -2)}}"] + + ["submission_date:DATE:{{ds}}"], + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 21:14:41.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 21:01:02.000000000 +0000 @@ -153,6 +153,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v2_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v2_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=68400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v2_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) + checks__warn_ga_derived__blogs_goals__v2 = bigquery_dq_check( task_id="checks__warn_ga_derived__blogs_goals__v2", source_table="blogs_goals_v2", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 20:56:54.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.events_v1` + `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- name: submission_date +- mode: NULLABLE + name: submission_date type: DATE - mode: NULLABLE -- name: source +- mode: NULLABLE + name: source type: STRING - mode: NULLABLE -- name: event_type +- mode: NULLABLE + name: event_type type: STRING - mode: NULLABLE -- name: form_factor +- mode: NULLABLE + name: form_factor type: STRING - mode: NULLABLE -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: subdivision1 +- mode: NULLABLE + name: subdivision1 type: STRING - mode: NULLABLE -- name: advertiser +- mode: NULLABLE + name: advertiser type: STRING - mode: NULLABLE -- name: release_channel +- mode: NULLABLE + name: release_channel type: STRING - mode: NULLABLE -- name: position +- mode: NULLABLE + name: position type: INTEGER - mode: NULLABLE -- name: provider +- mode: NULLABLE + name: provider type: STRING - mode: NULLABLE -- name: match_type +- mode: NULLABLE + name: match_type type: STRING - mode: NULLABLE -- name: normalized_os +- mode: NULLABLE + name: normalized_os type: STRING - mode: NULLABLE -- name: suggest_data_sharing_enabled +- mode: NULLABLE + name: suggest_data_sharing_enabled type: BOOLEAN - mode: NULLABLE -- name: event_count +- mode: NULLABLE + name: event_count type: INTEGER - mode: NULLABLE -- name: user_count +- mode: NULLABLE + name: user_count type: INTEGER - mode: NULLABLE -- name: query_type +- mode: NULLABLE + name: query_type type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- name: submission_date +- mode: NULLABLE + name: submission_date type: DATE - mode: NULLABLE -- name: form_factor +- mode: NULLABLE + name: form_factor type: STRING - mode: NULLABLE -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: advertiser +- mode: NULLABLE + name: advertiser type: STRING - mode: NULLABLE -- name: normalized_os +- mode: NULLABLE + name: normalized_os type: STRING - mode: NULLABLE -- name: release_channel +- mode: NULLABLE + name: release_channel type: STRING - mode: NULLABLE -- name: position +- mode: NULLABLE + name: position type: INTEGER - mode: NULLABLE -- name: provider +- mode: NULLABLE + name: provider type: STRING - mode: NULLABLE -- name: match_type +- mode: NULLABLE + name: match_type type: STRING - mode: NULLABLE -- name: suggest_data_sharing_enabled +- mode: NULLABLE + name: suggest_data_sharing_enabled type: BOOLEAN - mode: NULLABLE -- name: impression_count +- mode: NULLABLE + name: impression_count type: INTEGER - mode: NULLABLE -- name: click_count +- mode: NULLABLE + name: click_count type: INTEGER - mode: NULLABLE -- name: query_type +- mode: NULLABLE + name: query_type type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -26,9 +26,6 @@ - name: adjust_network type: STRING mode: NULLABLE -- name: install_source - type: STRING - mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -48,10 +48,6 @@ description: 'The type of source of a client installation. ' -- name: install_source - type: STRING - mode: NULLABLE - description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 20:56:54.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 20:56:54.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -80,7 +80,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: city +- mode: NULLABLE + name: city type: STRING - mode: NULLABLE -- name: datetime +- mode: NULLABLE + name: datetime type: TIMESTAMP - mode: NULLABLE -- name: proportion_undefined +- mode: NULLABLE + name: proportion_undefined type: FLOAT - mode: NULLABLE -- name: proportion_timeout +- mode: NULLABLE + name: proportion_timeout type: FLOAT - mode: NULLABLE -- name: proportion_abort +- mode: NULLABLE + name: proportion_abort type: FLOAT - mode: NULLABLE -- name: proportion_unreachable +- mode: NULLABLE + name: proportion_unreachable type: FLOAT - mode: NULLABLE -- name: proportion_terminated +- mode: NULLABLE + name: proportion_terminated type: FLOAT - mode: NULLABLE -- name: proportion_channel_open +- mode: NULLABLE + name: proportion_channel_open type: FLOAT - mode: NULLABLE -- name: avg_dns_success_time +- mode: NULLABLE + name: avg_dns_success_time type: FLOAT - mode: NULLABLE -- name: missing_dns_success +- mode: NULLABLE + name: missing_dns_success type: FLOAT - mode: NULLABLE -- name: avg_dns_failure_time +- mode: NULLABLE + name: avg_dns_failure_time type: FLOAT - mode: NULLABLE -- name: missing_dns_failure +- mode: NULLABLE + name: missing_dns_failure type: FLOAT - mode: NULLABLE -- name: count_dns_failure +- mode: NULLABLE + name: count_dns_failure type: FLOAT - mode: NULLABLE -- name: ssl_error_prop +- mode: NULLABLE + name: ssl_error_prop type: FLOAT - mode: NULLABLE -- name: avg_tls_handshake_time +- mode: NULLABLE + name: avg_tls_handshake_time type: FLOAT - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/active_users_aggregates_v3/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/active_users_aggregates_v3/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/active_users_aggregates_v3/query.sql 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/active_users_aggregates_v3/query.sql 2024-06-06 20:56:54.000000000 +0000 @@ -38,7 +38,7 @@ SAFE_OFFSET(0) ] AS normalized_channel, CAST(NULL AS INTEGER) AS uri_count, - CAST(NULL AS BOOL) AS is_default_browser, + CAST(NULL AS INTEGER) AS is_default_browser, FROM `moz-fx-data-shared-prod.klar_android.metrics_clients_last_seen` WHERE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-06 20:56:54.000000000 +0000 @@ -45,7 +45,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -75,7 +75,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -572,7 +572,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -592,7 +592,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -663,7 +663,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -683,7 +683,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -754,7 +754,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -774,7 +774,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1555,7 +1555,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1565,7 +1565,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1646,7 +1646,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1656,7 +1656,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1737,7 +1737,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1747,7 +1747,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1828,7 +1828,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1838,7 +1838,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1990,7 +1990,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -2010,7 +2010,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.events_v1` + `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2081,7 +2081,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` + `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` UNION ALL SELECT submission_timestamp, @@ -2091,7 +2091,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` + `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2162,7 +2162,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -2172,7 +2172,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 20:58:15.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 20:55:39.000000000 +0000 @@ -0,0 +1,10 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + activity_date AS activity_dt, + activity_datetime AS activity_date, + gclid, + conversion_name +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 20:58:12.000000000 +0000 @@ -0,0 +1,30 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed + date_partition_parameter: activity_date + date_partition_offset: -2 + parameters: + - submission_date:DATE:{{ds}} +bigquery: + time_partitioning: + type: day + field: run_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 20:55:39.000000000 +0000 @@ -0,0 +1,31 @@ +SELECT + activity_date, + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_datetime, + @submission_date AS run_date, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.gclid_conversions_v2` UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date = DATE_SUB(@activity_date, INTERVAL 2 DAY) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -0,0 +1,21 @@ +fields: +- name: activity_date + mode: NULLABLE + type: DATE + description: Activity Date +- name: activity_datetime + mode: NULLABLE + type: STRING + description: Activity Datetime +- name: run_date + mode: NULLABLE + type: DATE + description: Run Date - The submission date +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_vpn/main/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_vpn/main/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_vpn/main/metadata.yaml 2024-06-06 21:12:46.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_vpn/main/metadata.yaml 2024-06-06 20:56:54.000000000 +0000 @@ -2,7 +2,7 @@ description: |- This a view that UNIONs the stable ping tables across all channels of the Glean application "Mozilla VPN" - (mozillavpn.main, org_mozilla_firefox_vpn.main, org_mozilla_ios_firefoxvpn.main). + (mozillavpn.main, org_mozilla_firefox_vpn.main, org_mozilla_ios_firefoxvpn.main, org_mozilla_ios_firefoxvpn_network_extension.main). It is used by Looker. owners: [] @@ -17,3 +17,4 @@ - moz-fx-data-shared-prod.mozillavpn.main - moz-fx-data-shared-prod.org_mozilla_firefox_vpn.main - moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn.main + - moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension.main diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_vpn/main/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_vpn/main/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_vpn/main/view.sql 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_vpn/main/view.sql 2024-06-06 20:56:54.000000000 +0000 @@ -148,3 +148,98 @@ submission_timestamp FROM `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn.main` +UNION ALL +SELECT + "org_mozilla_ios_firefoxvpn_network_extension" AS normalized_app_id, + "release" AS normalized_channel, + additional_properties, + STRUCT( + client_info.android_sdk_version, + client_info.app_build, + client_info.app_channel, + client_info.app_display_version, + client_info.architecture, + client_info.client_id, + client_info.device_manufacturer, + client_info.device_model, + client_info.first_run_date, + client_info.locale, + client_info.os, + client_info.os_version, + client_info.telemetry_sdk_build, + client_info.build_date, + client_info.windows_build_number, + client_info.session_count, + client_info.session_id + ) AS `client_info`, + document_id, + events, + STRUCT( + metadata.geo, + STRUCT( + metadata.header.date, + metadata.header.dnt, + metadata.header.x_debug_id, + metadata.header.x_pingsender_version, + metadata.header.x_source_tags, + metadata.header.x_telemetry_agent, + metadata.header.x_foxsec_ip_reputation, + metadata.header.x_lb_tags, + metadata.header.parsed_date, + metadata.header.parsed_x_source_tags, + metadata.header.parsed_x_lb_tags + ) AS `header`, + metadata.isp, + metadata.user_agent + ) AS `metadata`, + STRUCT( + metrics.labeled_counter, + metrics.boolean, + CAST( + NULL + AS + STRUCT< + `performance_time_to_main_screen` STRUCT< + `bucket_count` INTEGER, + `count` INTEGER, + `histogram_type` STRING, + `overflow` INTEGER, + `range` ARRAY, + `sum` INTEGER, + `time_unit` STRING, + `underflow` INTEGER, + `values` ARRAY> + > + > + ) AS `timing_distribution`, + metrics.string + ) AS `metrics`, + normalized_app_name, + normalized_country_code, + normalized_os, + normalized_os_version, + STRUCT( + ping_info.end_time, + ARRAY( + SELECT + STRUCT( + experiments.key, + STRUCT( + experiments.value.branch, + STRUCT(experiments.value.extra.type, experiments.value.extra.enrollment_id) AS `extra` + ) AS `value` + ) + FROM + UNNEST(ping_info.experiments) AS `experiments` + ) AS `experiments`, + ping_info.ping_type, + ping_info.reason, + ping_info.seq, + ping_info.start_time, + ping_info.parsed_start_time, + ping_info.parsed_end_time + ) AS `ping_info`, + sample_id, + submission_timestamp +FROM + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension.main` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 20:56:54.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.main_v1` + `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.main_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -1,13 +1,7 @@ fields: -- name: build_hour - type: DATETIME - mode: NULLABLE - description: null -- name: geckoview_major_version - type: INTEGER - mode: NULLABLE - description: null -- name: n_pings - type: INTEGER - mode: NULLABLE - description: null +- type: DATETIME + name: build_hour +- type: INTEGER + name: geckoview_major_version +- type: INTEGER + name: n_pings diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 20:56:54.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 20:56:54.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 20:56:54.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 20:56:54.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 20:56:54.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 20:56:54.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -1,7 +1,9 @@ fields: - name: postal_district_code type: STRING - mode: NULLABLE + mode: REQUIRED + description: One-character Canadian postal district code. - name: province_code type: STRING mode: NULLABLE + description: Two-character Canadian province/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -1,28 +1,47 @@ fields: - name: name + description: Official country name per ISO 3166 type: STRING - mode: NULLABLE + mode: REQUIRED - name: code + description: ISO 3166 alpha-2 country code type: STRING - mode: NULLABLE + mode: REQUIRED - name: code_3 + description: ISO 3166 alpha-3 country code type: STRING - mode: NULLABLE + mode: REQUIRED - name: region_name + description: Region name. These are based on the UN Statistics Division standard + country or area codes for statistical use (M49), but with the "Americas" region + split into "North America" and "South America". type: STRING - mode: NULLABLE + mode: REQUIRED - name: subregion_name + description: Sub-region name. These are based on UN Statistics Division standard + country or area codes for statistical use (M49), but with the "Latin America and the + Caribbean" and "Sub-Saharan Africa" sub-regions split into more specific + sub-regions. type: STRING - mode: NULLABLE + mode: REQUIRED - name: pocket_available_on_newtab - type: BOOLEAN - mode: NULLABLE + description: Whether Pocket is available on the newtab page in this country. Note + that Pocket might only be available in certain locales/languages within a country. + type: BOOL + mode: REQUIRED - name: mozilla_vpn_available - type: BOOLEAN - mode: NULLABLE + description: Whether Mozilla VPN is available in this country. + type: BOOL + mode: REQUIRED - name: sponsored_tiles_available_on_newtab - type: BOOLEAN - mode: NULLABLE + description: Whether sponsored tiles are available on the newtab page in this country. + Note that Pocket might only be available in certain locales/languages within a + country. + type: BOOL + mode: REQUIRED - name: ads_value_tier + description: Lowercase label detailing the monetary value tier that Mozilla Ads + assign to that region based on market size and our existing products, e.g., tier + 1, tier 2, etc. type: STRING - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -1,7 +1,10 @@ fields: - name: name + description: An alias for a country's name (including misspellings and alternate + encodings). type: STRING - mode: NULLABLE + mode: REQUIRED - name: code + description: ISO 3166 alpha-2 country code type: STRING - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: incident +- mode: NULLABLE + name: incident type: STRING - mode: NULLABLE -- name: description +- mode: NULLABLE + name: description type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE -- name: product +- mode: NULLABLE + name: product type: STRING - mode: NULLABLE -- name: version +- mode: NULLABLE + name: version type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -1,16 +1,27 @@ fields: -- name: value +- mode: NULLABLE + description: Hex value assigned to the TLS cipher, in format like "0x00,0x84"; note + some values are ranges or contain wildcards + name: value type: STRING - mode: NULLABLE -- name: description +- mode: NULLABLE + description: Human-readable name of the TLS cipher + name: description type: STRING - mode: NULLABLE -- name: dtls_ok +- mode: NULLABLE + description: Any TLS cipher suite that is specified for use with DTLS MUST define + limits on the use of the associated AEAD function that preserves margins for both + confidentiality and integrity, as specified in [RFC-ietf-tls-dtls13-43] + name: dtls_ok type: BOOLEAN - mode: NULLABLE -- name: recommended +- mode: NULLABLE + description: Whether the TLS cipher is recommended by the IETF. If an item is not + marked as "recommended", it does not necessarily mean that it is flawed; rather, + it indicates that the item either has not been through the IETF consensus process, + has limited applicability, or is intended only for specific use cases + name: recommended type: BOOLEAN - mode: NULLABLE -- name: reference +- mode: NULLABLE + description: RFCs or associated reference material for the TLS cipher + name: reference type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -1,13 +1,17 @@ fields: - name: code_3 + description: ISO 639 alpha-3 language code. type: STRING - mode: NULLABLE + mode: REQUIRED - name: code_2 + description: ISO 639 alpha-2 language code (if any). type: STRING mode: NULLABLE - name: name + description: Language name. type: STRING - mode: NULLABLE + mode: REQUIRED - name: other_names + description: Other names for the language (if any). type: STRING mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -1,19 +1,19 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -1,25 +1,25 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: document_version +- mode: NULLABLE + name: document_version type: STRING - mode: NULLABLE -- name: path +- mode: NULLABLE + name: path type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: document_version +- mode: NULLABLE + name: document_version type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: path +- mode: NULLABLE + name: path type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -1,7 +1,7 @@ fields: - name: score type: NUMERIC - mode: NULLABLE + mode: REQUIRED - name: value type: NUMERIC - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-06 20:55:39.000000000 +0000 @@ -1,7 +1,9 @@ fields: - name: zip_code_prefix type: STRING - mode: NULLABLE + mode: REQUIRED + description: Three-digit US ZIP code prefix. - name: state_code type: STRING mode: NULLABLE + description: Two-character US state/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users_aggregates_mobile/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users_aggregates_mobile/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users_aggregates_mobile/view.sql 2024-06-06 21:11:43.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users_aggregates_mobile/view.sql 2024-06-06 20:56:54.000000000 +0000 @@ -3,223 +3,31 @@ `moz-fx-data-shared-prod.telemetry.active_users_aggregates_mobile` AS SELECT - segment, - attribution_medium, - attribution_source, - attributed, - city, - country, - distribution_id, - first_seen_year, - is_default_browser, - locale, - channel, - os, - os_version, - os_version_major, - os_version_minor, - submission_date, - adjust_network, - install_source, - daily_users, - weekly_users, - monthly_users, - dau, - wau, - mau, - uri_count, - active_hours, - app_name, - app_version, - app_version_major, - app_version_minor, - app_version_patch_revision, - app_version_is_major_release, - os_grouped, + * FROM `moz-fx-data-shared-prod.fenix.active_users_aggregates` UNION ALL SELECT - segment, - attribution_medium, - attribution_source, - attributed, - city, - country, - distribution_id, - first_seen_year, - is_default_browser, - locale, - channel, - os, - os_version, - os_version_major, - os_version_minor, - submission_date, - adjust_network, - install_source, - daily_users, - weekly_users, - monthly_users, - dau, - wau, - mau, - uri_count, - active_hours, - app_name, - app_version, - app_version_major, - app_version_minor, - app_version_patch_revision, - app_version_is_major_release, - os_grouped, + * FROM `moz-fx-data-shared-prod.firefox_ios.active_users_aggregates` UNION ALL SELECT - segment, - attribution_medium, - attribution_source, - attributed, - city, - country, - distribution_id, - first_seen_year, - is_default_browser, - locale, - channel, - os, - os_version, - os_version_major, - os_version_minor, - submission_date, - adjust_network, - install_source, - daily_users, - weekly_users, - monthly_users, - dau, - wau, - mau, - uri_count, - active_hours, - app_name, - app_version, - app_version_major, - app_version_minor, - app_version_patch_revision, - app_version_is_major_release, - os_grouped, + * FROM `moz-fx-data-shared-prod.focus_ios.active_users_aggregates` UNION ALL SELECT - segment, - attribution_medium, - attribution_source, - attributed, - city, - country, - distribution_id, - first_seen_year, - is_default_browser, - locale, - channel, - os, - os_version, - os_version_major, - os_version_minor, - submission_date, - adjust_network, - install_source, - daily_users, - weekly_users, - monthly_users, - dau, - wau, - mau, - uri_count, - active_hours, - app_name, - app_version, - app_version_major, - app_version_minor, - app_version_patch_revision, - app_version_is_major_release, - os_grouped, + * FROM `moz-fx-data-shared-prod.klar_ios.active_users_aggregates` UNION ALL SELECT - segment, - attribution_medium, - attribution_source, - attributed, - city, - country, - distribution_id, - first_seen_year, - is_default_browser, - locale, - channel, - os, - os_version, - os_version_major, - os_version_minor, - submission_date, - adjust_network, - install_source, - daily_users, - weekly_users, - monthly_users, - dau, - wau, - mau, - uri_count, - active_hours, - app_name, - app_version, - app_version_major, - app_version_minor, - app_version_patch_revision, - app_version_is_major_release, - os_grouped, + * FROM `moz-fx-data-shared-prod.focus_android.active_users_aggregates` UNION ALL SELECT - segment, - attribution_medium, - attribution_source, - attributed, - city, - country, - distribution_id, - first_seen_year, - is_default_browser, - locale, - channel, - os, - os_version, - os_version_major, - os_version_minor, - submission_date, - adjust_network, - install_source, - daily_users, - weekly_users, ```

⚠️ Only part of the diff is displayed.

Link to full diff

dataops-ci-bot commented 4 weeks ago

Integration report for "GROWTH-143 add new bqetl feed for census"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 21:08:26.000000000 +0000 @@ -0,0 +1,82 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_google_analytics_derived_ga4", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_delta=datetime.timedelta(seconds=18000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table='ga_desktop_conversions_v1${{ macros.ds_format(macros.ds_add(ds, -2), "%Y-%m-%d", "%Y%m%d") }}', + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + parameters=["activity_date:DATE:{{macros.ds_add(ds, -2)}}"] + + ["submission_date:DATE:{{ds}}"], + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 21:21:51.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 21:08:26.000000000 +0000 @@ -153,6 +153,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v2_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v2_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=68400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v2_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) + checks__warn_ga_derived__blogs_goals__v2 = bigquery_dq_check( task_id="checks__warn_ga_derived__blogs_goals__v2", source_table="blogs_goals_v2", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- name: submission_date +- mode: NULLABLE + name: submission_date type: DATE - mode: NULLABLE -- name: source +- mode: NULLABLE + name: source type: STRING - mode: NULLABLE -- name: event_type +- mode: NULLABLE + name: event_type type: STRING - mode: NULLABLE -- name: form_factor +- mode: NULLABLE + name: form_factor type: STRING - mode: NULLABLE -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: subdivision1 +- mode: NULLABLE + name: subdivision1 type: STRING - mode: NULLABLE -- name: advertiser +- mode: NULLABLE + name: advertiser type: STRING - mode: NULLABLE -- name: release_channel +- mode: NULLABLE + name: release_channel type: STRING - mode: NULLABLE -- name: position +- mode: NULLABLE + name: position type: INTEGER - mode: NULLABLE -- name: provider +- mode: NULLABLE + name: provider type: STRING - mode: NULLABLE -- name: match_type +- mode: NULLABLE + name: match_type type: STRING - mode: NULLABLE -- name: normalized_os +- mode: NULLABLE + name: normalized_os type: STRING - mode: NULLABLE -- name: suggest_data_sharing_enabled +- mode: NULLABLE + name: suggest_data_sharing_enabled type: BOOLEAN - mode: NULLABLE -- name: event_count +- mode: NULLABLE + name: event_count type: INTEGER - mode: NULLABLE -- name: user_count +- mode: NULLABLE + name: user_count type: INTEGER - mode: NULLABLE -- name: query_type +- mode: NULLABLE + name: query_type type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -1,40 +1,40 @@ fields: -- name: submission_date +- mode: NULLABLE + name: submission_date type: DATE - mode: NULLABLE -- name: form_factor +- mode: NULLABLE + name: form_factor type: STRING - mode: NULLABLE -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: advertiser +- mode: NULLABLE + name: advertiser type: STRING - mode: NULLABLE -- name: normalized_os +- mode: NULLABLE + name: normalized_os type: STRING - mode: NULLABLE -- name: release_channel +- mode: NULLABLE + name: release_channel type: STRING - mode: NULLABLE -- name: position +- mode: NULLABLE + name: position type: INTEGER - mode: NULLABLE -- name: provider +- mode: NULLABLE + name: provider type: STRING - mode: NULLABLE -- name: match_type +- mode: NULLABLE + name: match_type type: STRING - mode: NULLABLE -- name: suggest_data_sharing_enabled +- mode: NULLABLE + name: suggest_data_sharing_enabled type: BOOLEAN - mode: NULLABLE -- name: impression_count +- mode: NULLABLE + name: impression_count type: INTEGER - mode: NULLABLE -- name: click_count +- mode: NULLABLE + name: click_count type: INTEGER - mode: NULLABLE -- name: query_type +- mode: NULLABLE + name: query_type type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -26,9 +26,6 @@ - name: adjust_network type: STRING mode: NULLABLE -- name: install_source - type: STRING - mode: NULLABLE - name: retained_week_2 type: BOOLEAN mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -48,10 +48,6 @@ description: 'The type of source of a client installation. ' -- name: install_source - type: STRING - mode: NULLABLE - description: null - name: new_profiles type: INTEGER mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:04:23.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:04:23.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -80,7 +80,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/internet_outages/global_outages_v1/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -1,49 +1,49 @@ fields: -- name: country +- mode: NULLABLE + name: country type: STRING - mode: NULLABLE -- name: city +- mode: NULLABLE + name: city type: STRING - mode: NULLABLE -- name: datetime +- mode: NULLABLE + name: datetime type: TIMESTAMP - mode: NULLABLE -- name: proportion_undefined +- mode: NULLABLE + name: proportion_undefined type: FLOAT - mode: NULLABLE -- name: proportion_timeout +- mode: NULLABLE + name: proportion_timeout type: FLOAT - mode: NULLABLE -- name: proportion_abort +- mode: NULLABLE + name: proportion_abort type: FLOAT - mode: NULLABLE -- name: proportion_unreachable +- mode: NULLABLE + name: proportion_unreachable type: FLOAT - mode: NULLABLE -- name: proportion_terminated +- mode: NULLABLE + name: proportion_terminated type: FLOAT - mode: NULLABLE -- name: proportion_channel_open +- mode: NULLABLE + name: proportion_channel_open type: FLOAT - mode: NULLABLE -- name: avg_dns_success_time +- mode: NULLABLE + name: avg_dns_success_time type: FLOAT - mode: NULLABLE -- name: missing_dns_success +- mode: NULLABLE + name: missing_dns_success type: FLOAT - mode: NULLABLE -- name: avg_dns_failure_time +- mode: NULLABLE + name: avg_dns_failure_time type: FLOAT - mode: NULLABLE -- name: missing_dns_failure +- mode: NULLABLE + name: missing_dns_failure type: FLOAT - mode: NULLABLE -- name: count_dns_failure +- mode: NULLABLE + name: count_dns_failure type: FLOAT - mode: NULLABLE -- name: ssl_error_prop +- mode: NULLABLE + name: ssl_error_prop type: FLOAT - mode: NULLABLE -- name: avg_tls_handshake_time +- mode: NULLABLE + name: avg_tls_handshake_time type: FLOAT - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/active_users_aggregates_v3/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/active_users_aggregates_v3/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/active_users_aggregates_v3/query.sql 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/active_users_aggregates_v3/query.sql 2024-06-06 21:04:23.000000000 +0000 @@ -38,7 +38,7 @@ SAFE_OFFSET(0) ] AS normalized_channel, CAST(NULL AS INTEGER) AS uri_count, - CAST(NULL AS BOOL) AS is_default_browser, + CAST(NULL AS INTEGER) AS is_default_browser, FROM `moz-fx-data-shared-prod.klar_android.metrics_clients_last_seen` WHERE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-06 21:04:23.000000000 +0000 @@ -45,7 +45,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -75,7 +75,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -572,7 +572,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -582,7 +582,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -663,7 +663,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -673,7 +673,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -754,7 +754,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -764,7 +764,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -1565,7 +1565,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1575,7 +1575,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1656,7 +1656,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1666,7 +1666,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1747,7 +1747,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1757,7 +1757,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1838,7 +1838,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1848,7 +1848,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2081,7 +2081,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` + `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` UNION ALL SELECT submission_timestamp, @@ -2091,7 +2091,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` + `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2162,7 +2162,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -2172,7 +2172,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 21:05:24.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 21:03:36.000000000 +0000 @@ -0,0 +1,10 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + activity_date AS activity_dt, + activity_datetime AS activity_date, + gclid, + conversion_name +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 21:05:28.000000000 +0000 @@ -0,0 +1,30 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed + date_partition_parameter: activity_date + date_partition_offset: -2 + parameters: + - submission_date:DATE:{{ds}} +bigquery: + time_partitioning: + type: day + field: null + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 21:03:36.000000000 +0000 @@ -0,0 +1,31 @@ +SELECT + activity_date, + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_datetime, + @submission_date AS run_date, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.gclid_conversions_v2` UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date = @activity_date diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -0,0 +1,21 @@ +fields: +- name: activity_date + mode: NULLABLE + type: DATE + description: Activity Date +- name: activity_datetime + mode: NULLABLE + type: STRING + description: Activity Datetime +- name: run_date + mode: NULLABLE + type: DATE + description: Run Date - The submission date +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:04:23.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.main_v1` + `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_fenix/geckoview_version/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -1,13 +1,7 @@ fields: -- name: build_hour - type: DATETIME - mode: NULLABLE - description: null -- name: geckoview_major_version - type: INTEGER - mode: NULLABLE - description: null -- name: n_pings - type: INTEGER - mode: NULLABLE - description: null +- type: DATETIME + name: build_hour +- type: INTEGER + name: geckoview_major_version +- type: INTEGER + name: n_pings diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:18:38.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:04:23.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:04:23.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:18:38.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:04:23.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:04:23.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:04:23.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:04:23.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/ca_postal_districts_v1/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -1,7 +1,9 @@ fields: - name: postal_district_code type: STRING - mode: NULLABLE + mode: REQUIRED + description: One-character Canadian postal district code. - name: province_code type: STRING mode: NULLABLE + description: Two-character Canadian province/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_codes_v1/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -1,28 +1,47 @@ fields: - name: name + description: Official country name per ISO 3166 type: STRING - mode: NULLABLE + mode: REQUIRED - name: code + description: ISO 3166 alpha-2 country code type: STRING - mode: NULLABLE + mode: REQUIRED - name: code_3 + description: ISO 3166 alpha-3 country code type: STRING - mode: NULLABLE + mode: REQUIRED - name: region_name + description: Region name. These are based on the UN Statistics Division standard + country or area codes for statistical use (M49), but with the "Americas" region + split into "North America" and "South America". type: STRING - mode: NULLABLE + mode: REQUIRED - name: subregion_name + description: Sub-region name. These are based on UN Statistics Division standard + country or area codes for statistical use (M49), but with the "Latin America and the + Caribbean" and "Sub-Saharan Africa" sub-regions split into more specific + sub-regions. type: STRING - mode: NULLABLE + mode: REQUIRED - name: pocket_available_on_newtab - type: BOOLEAN - mode: NULLABLE + description: Whether Pocket is available on the newtab page in this country. Note + that Pocket might only be available in certain locales/languages within a country. + type: BOOL + mode: REQUIRED - name: mozilla_vpn_available - type: BOOLEAN - mode: NULLABLE + description: Whether Mozilla VPN is available in this country. + type: BOOL + mode: REQUIRED - name: sponsored_tiles_available_on_newtab - type: BOOLEAN - mode: NULLABLE + description: Whether sponsored tiles are available on the newtab page in this country. + Note that Pocket might only be available in certain locales/languages within a + country. + type: BOOL + mode: REQUIRED - name: ads_value_tier + description: Lowercase label detailing the monetary value tier that Mozilla Ads + assign to that region based on market size and our existing products, e.g., tier + 1, tier 2, etc. type: STRING - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/country_names_v1/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -1,7 +1,10 @@ fields: - name: name + description: An alias for a country's name (including misspellings and alternate + encodings). type: STRING - mode: NULLABLE + mode: REQUIRED - name: code + description: ISO 3166 alpha-2 country code type: STRING - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/data_incidents_v1/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: incident +- mode: NULLABLE + name: incident type: STRING - mode: NULLABLE -- name: description +- mode: NULLABLE + name: description type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE -- name: product +- mode: NULLABLE + name: product type: STRING - mode: NULLABLE -- name: version +- mode: NULLABLE + name: version type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/iana_tls_cipher_suites/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -1,16 +1,27 @@ fields: -- name: value +- mode: NULLABLE + description: Hex value assigned to the TLS cipher, in format like "0x00,0x84"; note + some values are ranges or contain wildcards + name: value type: STRING - mode: NULLABLE -- name: description +- mode: NULLABLE + description: Human-readable name of the TLS cipher + name: description type: STRING - mode: NULLABLE -- name: dtls_ok +- mode: NULLABLE + description: Any TLS cipher suite that is specified for use with DTLS MUST define + limits on the use of the associated AEAD function that preserves margins for both + confidentiality and integrity, as specified in [RFC-ietf-tls-dtls13-43] + name: dtls_ok type: BOOLEAN - mode: NULLABLE -- name: recommended +- mode: NULLABLE + description: Whether the TLS cipher is recommended by the IETF. If an item is not + marked as "recommended", it does not necessarily mean that it is flawed; rather, + it indicates that the item either has not been through the IETF consensus process, + has limited applicability, or is intended only for specific use cases + name: recommended type: BOOLEAN - mode: NULLABLE -- name: reference +- mode: NULLABLE + description: RFCs or associated reference material for the TLS cipher + name: reference type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/language_codes_v1/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -1,13 +1,17 @@ fields: - name: code_3 + description: ISO 639 alpha-3 language code. type: STRING - mode: NULLABLE + mode: REQUIRED - name: code_2 + description: ISO 639 alpha-2 language code (if any). type: STRING mode: NULLABLE - name: name + description: Language name. type: STRING - mode: NULLABLE + mode: REQUIRED - name: other_names + description: Other names for the language (if any). type: STRING mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_distinct_docids_notes_v1/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -1,19 +1,19 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_columns_notes_v1/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -1,25 +1,25 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: document_version +- mode: NULLABLE + name: document_version type: STRING - mode: NULLABLE -- name: path +- mode: NULLABLE + name: path type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_missing_document_namespaces_notes_v1/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: document_version +- mode: NULLABLE + name: document_version type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/monitoring_schema_errors_notes_v1/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -1,22 +1,22 @@ fields: -- name: start_date +- mode: NULLABLE + name: start_date type: DATE - mode: NULLABLE -- name: end_date +- mode: NULLABLE + name: end_date type: DATE - mode: NULLABLE -- name: document_namespace +- mode: NULLABLE + name: document_namespace type: STRING - mode: NULLABLE -- name: document_type +- mode: NULLABLE + name: document_type type: STRING - mode: NULLABLE -- name: path +- mode: NULLABLE + name: path type: STRING - mode: NULLABLE -- name: notes +- mode: NULLABLE + name: notes type: STRING - mode: NULLABLE -- name: bug +- mode: NULLABLE + name: bug type: STRING - mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/normal_distribution/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -1,7 +1,7 @@ fields: - name: score type: NUMERIC - mode: NULLABLE + mode: REQUIRED - name: value type: NUMERIC - mode: NULLABLE + mode: REQUIRED diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/static/us_zip_code_prefixes_v1/schema.yaml 2024-06-06 21:03:36.000000000 +0000 @@ -1,7 +1,9 @@ fields: - name: zip_code_prefix type: STRING - mode: NULLABLE + mode: REQUIRED + description: Three-digit US ZIP code prefix. - name: state_code type: STRING mode: NULLABLE + description: Two-character US state/territory code (if any). diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users_aggregates_mobile/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users_aggregates_mobile/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users_aggregates_mobile/view.sql 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/active_users_aggregates_mobile/view.sql 2024-06-06 21:04:23.000000000 +0000 @@ -3,223 +3,31 @@ `moz-fx-data-shared-prod.telemetry.active_users_aggregates_mobile` AS SELECT - segment, - attribution_medium, - attribution_source, - attributed, - city, - country, - distribution_id, - first_seen_year, - is_default_browser, - locale, - channel, - os, - os_version, - os_version_major, - os_version_minor, - submission_date, - adjust_network, - install_source, - daily_users, - weekly_users, - monthly_users, - dau, - wau, - mau, - uri_count, - active_hours, - app_name, - app_version, - app_version_major, - app_version_minor, - app_version_patch_revision, - app_version_is_major_release, - os_grouped, + * FROM `moz-fx-data-shared-prod.fenix.active_users_aggregates` UNION ALL SELECT - segment, - attribution_medium, - attribution_source, - attributed, - city, - country, - distribution_id, - first_seen_year, - is_default_browser, - locale, - channel, - os, - os_version, - os_version_major, - os_version_minor, - submission_date, - adjust_network, - install_source, - daily_users, - weekly_users, - monthly_users, - dau, - wau, - mau, - uri_count, - active_hours, - app_name, - app_version, - app_version_major, - app_version_minor, - app_version_patch_revision, - app_version_is_major_release, - os_grouped, + * FROM `moz-fx-data-shared-prod.firefox_ios.active_users_aggregates` UNION ALL SELECT - segment, - attribution_medium, - attribution_source, - attributed, - city, - country, - distribution_id, - first_seen_year, - is_default_browser, - locale, - channel, - os, - os_version, - os_version_major, - os_version_minor, - submission_date, - adjust_network, - install_source, - daily_users, - weekly_users, - monthly_users, - dau, - wau, - mau, - uri_count, - active_hours, - app_name, - app_version, - app_version_major, - app_version_minor, - app_version_patch_revision, - app_version_is_major_release, - os_grouped, + * FROM `moz-fx-data-shared-prod.focus_ios.active_users_aggregates` UNION ALL SELECT - segment, - attribution_medium, - attribution_source, - attributed, - city, - country, - distribution_id, - first_seen_year, - is_default_browser, - locale, - channel, - os, - os_version, - os_version_major, - os_version_minor, - submission_date, - adjust_network, - install_source, - daily_users, - weekly_users, - monthly_users, - dau, - wau, - mau, - uri_count, - active_hours, - app_name, - app_version, - app_version_major, - app_version_minor, - app_version_patch_revision, - app_version_is_major_release, - os_grouped, + * FROM `moz-fx-data-shared-prod.klar_ios.active_users_aggregates` UNION ALL SELECT - segment, - attribution_medium, - attribution_source, - attributed, - city, - country, - distribution_id, - first_seen_year, - is_default_browser, - locale, - channel, - os, - os_version, - os_version_major, - os_version_minor, - submission_date, - adjust_network, - install_source, - daily_users, - weekly_users, - monthly_users, - dau, - wau, - mau, - uri_count, - active_hours, - app_name, - app_version, - app_version_major, - app_version_minor, - app_version_patch_revision, - app_version_is_major_release, - os_grouped, + * FROM `moz-fx-data-shared-prod.focus_android.active_users_aggregates` UNION ALL SELECT - segment, - attribution_medium, - attribution_source, - attributed, - city, - country, - distribution_id, - first_seen_year, - is_default_browser, - locale, - channel, - os, - os_version, - os_version_major, - os_version_minor, - submission_date, - adjust_network, - install_source, - daily_users, - weekly_users, - monthly_users, - dau, - wau, - mau, - uri_count, - active_hours, - app_name, - app_version, - app_version_major, - app_version_minor, - app_version_patch_revision, - app_version_is_major_release, - os_grouped, + * FROM `moz-fx-data-shared-prod.klar_android.active_users_aggregates` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml 2024-06-06 21:04:23.000000000 +0000 @@ -2,21 +2,18 @@ - name: date type: DATE mode: NULLABLE - description: null - name: product type: STRING mode: NULLABLE - name: category type: STRING mode: NULLABLE - description: null - name: channel type: STRING mode: NULLABLE - name: build_number type: INTEGER mode: NULLABLE - description: null - name: release_date type: DATE mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:18:37.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:04:23.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_live.main_events_v1` + `moz-fx-data-shared-prod.viu_politica_live.video_index_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_live.video_index_v1` + `moz-fx-data-shared-prod.viu_politica_live.main_events_v1` ) CROSS JOIN UNNEST(events) AS event, ```

Link to full diff

dataops-ci-bot commented 4 weeks ago

Integration report for "Merge branch 'main' into census_desktop_conv_evnt_feed"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 21:22:31.000000000 +0000 @@ -0,0 +1,82 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_google_analytics_derived_ga4", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_delta=datetime.timedelta(seconds=18000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table='ga_desktop_conversions_v1${{ macros.ds_format(macros.ds_add(ds, -2), "%Y-%m-%d", "%Y%m%d") }}', + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + parameters=["activity_date:DATE:{{macros.ds_add(ds, -2)}}"] + + ["submission_date:DATE:{{ds}}"], + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 21:22:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 21:22:31.000000000 +0000 @@ -153,6 +153,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v2_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v2_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=68400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v2_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) + checks__warn_ga_derived__blogs_goals__v2 = bigquery_dq_check( task_id="checks__warn_ga_derived__blogs_goals__v2", source_table="blogs_goals_v2", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:19:11.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:06:54.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:19:11.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:06:55.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -80,7 +80,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-06 21:19:11.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-06 21:08:30.000000000 +0000 @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` UNION ALL SELECT submission_timestamp, @@ -75,7 +75,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -572,7 +572,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -582,7 +582,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -663,7 +663,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -673,7 +673,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -754,7 +754,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -764,7 +764,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -2081,7 +2081,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` + `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` UNION ALL SELECT submission_timestamp, @@ -2091,7 +2091,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` + `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2162,7 +2162,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -2172,7 +2172,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 21:19:35.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 21:05:09.000000000 +0000 @@ -0,0 +1,10 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + activity_date AS activity_dt, + activity_datetime AS activity_date, + gclid, + conversion_name +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 21:19:35.000000000 +0000 @@ -0,0 +1,30 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed + date_partition_parameter: activity_date + date_partition_offset: -2 + parameters: + - submission_date:DATE:{{ds}} +bigquery: + time_partitioning: + type: day + field: null + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 21:05:09.000000000 +0000 @@ -0,0 +1,31 @@ +SELECT + activity_date, + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_datetime, + @submission_date AS run_date, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.gclid_conversions_v2` UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date = @activity_date diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 21:05:09.000000000 +0000 @@ -0,0 +1,21 @@ +fields: +- name: activity_date + mode: NULLABLE + type: DATE + description: Activity Date +- name: activity_datetime + mode: NULLABLE + type: STRING + description: Activity Datetime +- name: run_date + mode: NULLABLE + type: DATE + description: Run Date - The submission date +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:19:10.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:06:56.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:19:11.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:06:56.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:19:10.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:06:56.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.first_session_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml 2024-06-06 21:19:11.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml 2024-06-06 21:14:53.000000000 +0000 @@ -2,21 +2,18 @@ - name: date type: DATE mode: NULLABLE - description: null - name: product type: STRING mode: NULLABLE - name: category type: STRING mode: NULLABLE - description: null - name: channel type: STRING mode: NULLABLE - name: build_number type: INTEGER mode: NULLABLE - description: null - name: release_date type: DATE mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:19:10.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:06:54.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_live.video_index_v1` + `moz-fx-data-shared-prod.viu_politica_live.main_events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_live.main_events_v1` + `moz-fx-data-shared-prod.viu_politica_live.video_index_v1` ) CROSS JOIN UNNEST(events) AS event, ```

Link to full diff

dataops-ci-bot commented 4 weeks ago

Integration report for "Update partition by field"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 21:24:09.000000000 +0000 @@ -0,0 +1,82 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_google_analytics_derived_ga4", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_delta=datetime.timedelta(seconds=18000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table='ga_desktop_conversions_v1${{ macros.ds_format(macros.ds_add(ds, -2), "%Y-%m-%d", "%Y%m%d") }}', + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + parameters=["activity_date:DATE:{{macros.ds_add(ds, -2)}}"] + + ["submission_date:DATE:{{ds}}"], + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 21:23:09.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 21:24:09.000000000 +0000 @@ -153,6 +153,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v2_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v2_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=68400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v2_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) + checks__warn_ga_derived__blogs_goals__v2 = bigquery_dq_check( task_id="checks__warn_ga_derived__blogs_goals__v2", source_table="blogs_goals_v2", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 21:21:23.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 21:19:38.000000000 +0000 @@ -0,0 +1,10 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + activity_date AS activity_dt, + activity_datetime AS activity_date, + gclid, + conversion_name +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 21:21:23.000000000 +0000 @@ -0,0 +1,30 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed + date_partition_parameter: activity_date + date_partition_offset: -2 + parameters: + - submission_date:DATE:{{ds}} +bigquery: + time_partitioning: + type: day + field: activity_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 21:19:38.000000000 +0000 @@ -0,0 +1,31 @@ +SELECT + activity_date, + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_datetime, + @submission_date AS run_date, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.gclid_conversions_v2` UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date = @activity_date diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 21:19:38.000000000 +0000 @@ -0,0 +1,21 @@ +fields: +- name: activity_date + mode: NULLABLE + type: DATE + description: Activity Date +- name: activity_datetime + mode: NULLABLE + type: STRING + description: Activity Datetime +- name: run_date + mode: NULLABLE + type: DATE + description: Run Date - The submission date +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name ```

Link to full diff

dataops-ci-bot commented 4 weeks ago

Integration report for "Update view.sql"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 21:28:31.000000000 +0000 @@ -0,0 +1,82 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_google_analytics_derived_ga4", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_delta=datetime.timedelta(seconds=18000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table='ga_desktop_conversions_v1${{ macros.ds_format(macros.ds_add(ds, -2), "%Y-%m-%d", "%Y%m%d") }}', + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + parameters=["activity_date:DATE:{{macros.ds_add(ds, -2)}}"] + + ["submission_date:DATE:{{ds}}"], + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 21:27:08.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 21:28:31.000000000 +0000 @@ -153,6 +153,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v2_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v2_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=68400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v2_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) + checks__warn_ga_derived__blogs_goals__v2 = bigquery_dq_check( task_id="checks__warn_ga_derived__blogs_goals__v2", source_table="blogs_goals_v2", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 21:25:26.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 21:23:35.000000000 +0000 @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 21:25:25.000000000 +0000 @@ -0,0 +1,30 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed + date_partition_parameter: activity_date + date_partition_offset: -2 + parameters: + - submission_date:DATE:{{ds}} +bigquery: + time_partitioning: + type: day + field: activity_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 21:23:35.000000000 +0000 @@ -0,0 +1,31 @@ +SELECT + activity_date, + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_datetime, + @submission_date AS run_date, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.gclid_conversions_v2` UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date = @activity_date diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 21:23:35.000000000 +0000 @@ -0,0 +1,21 @@ +fields: +- name: activity_date + mode: NULLABLE + type: DATE + description: Activity Date +- name: activity_datetime + mode: NULLABLE + type: STRING + description: Activity Datetime +- name: run_date + mode: NULLABLE + type: DATE + description: Run Date - The submission date +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name ```

Link to full diff

dataops-ci-bot commented 4 weeks ago

Integration report for "Update metadata.yaml"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 21:30:57.000000000 +0000 @@ -0,0 +1,82 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_google_analytics_derived_ga4", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_delta=datetime.timedelta(seconds=18000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table='ga_desktop_conversions_v1${{ macros.ds_format(macros.ds_add(ds, -2), "%Y-%m-%d", "%Y%m%d") }}', + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + parameters=["activity_date:DATE:{{macros.ds_add(ds, -2)}}"] + + ["submission_date:DATE:{{ds}}"], + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 21:30:32.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 21:30:57.000000000 +0000 @@ -153,6 +153,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v2_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v2_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=68400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v2_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) + checks__warn_ga_derived__blogs_goals__v2 = bigquery_dq_check( task_id="checks__warn_ga_derived__blogs_goals__v2", source_table="blogs_goals_v2", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:27:40.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:15:38.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.events_v1` + `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:27:40.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:15:39.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -80,7 +80,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-06 21:27:40.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-06 21:17:12.000000000 +0000 @@ -45,7 +45,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` UNION ALL SELECT submission_timestamp, @@ -75,7 +75,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.prototype_no_code_events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1555,7 +1555,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1565,7 +1565,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1575,7 +1575,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1646,7 +1646,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1656,7 +1656,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1666,7 +1666,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1737,7 +1737,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1747,7 +1747,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1757,7 +1757,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1828,7 +1828,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` UNION ALL SELECT submission_timestamp, @@ -1838,7 +1838,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1848,7 +1848,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1990,7 +1990,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -2000,7 +2000,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.events_v1` + `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` UNION ALL SELECT submission_timestamp, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 21:28:12.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 21:13:53.000000000 +0000 @@ -0,0 +1,10 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + activity_date AS activity_dt, + activity_datetime AS activity_date, + gclid, + conversion_name +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 21:28:12.000000000 +0000 @@ -0,0 +1,30 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed + date_partition_parameter: activity_date + date_partition_offset: -2 + parameters: + - submission_date:DATE:{{ds}} +bigquery: + time_partitioning: + type: day + field: run_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 21:13:53.000000000 +0000 @@ -0,0 +1,31 @@ +SELECT + activity_date, + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_datetime, + @submission_date AS run_date, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.gclid_conversions_v2` UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date = @activity_date diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 21:13:53.000000000 +0000 @@ -0,0 +1,21 @@ +fields: +- name: activity_date + mode: NULLABLE + type: DATE + description: Activity Date +- name: activity_datetime + mode: NULLABLE + type: STRING + description: Activity Datetime +- name: run_date + mode: NULLABLE + type: DATE + description: Run Date - The submission date +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:27:40.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:15:38.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.main_v1` + `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:27:40.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:15:40.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:27:40.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:15:41.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:27:40.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:15:41.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` ) CROSS JOIN UNNEST(events) AS event, ```

Link to full diff

dataops-ci-bot commented 4 weeks ago

Integration report for "Merge branch 'main' into census_desktop_conv_evnt_feed"

sql.diff

Click to expand! ```diff Only in /tmp/workspace/generated-sql/dags/: bqetl_census_feed.py diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py /tmp/workspace/generated-sql/dags/bqetl_census_feed.py --- /tmp/workspace/main-generated-sql/dags/bqetl_census_feed.py 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_census_feed.py 2024-06-06 21:53:13.000000000 +0000 @@ -0,0 +1,82 @@ +# Generated via https://github.com/mozilla/bigquery-etl/blob/main/bigquery_etl/query_scheduling/generate_airflow_dags.py + +from airflow import DAG +from airflow.sensors.external_task import ExternalTaskMarker +from airflow.sensors.external_task import ExternalTaskSensor +from airflow.utils.task_group import TaskGroup +import datetime +from operators.gcp_container_operator import GKEPodOperator +from utils.constants import ALLOWED_STATES, FAILED_STATES +from utils.gcp import bigquery_etl_query, bigquery_dq_check + +docs = """ +### bqetl_census_feed + +Built from bigquery-etl repo, [`dags/bqetl_census_feed.py`](https://github.com/mozilla/bigquery-etl/blob/generated-sql/dags/bqetl_census_feed.py) + +#### Description + +Loads the desktop conversion event tables +#### Owner + +kwindau@mozilla.com + +#### Tags + +* impact/tier_2 +* repo/bigquery-etl +""" + + +default_args = { + "owner": "kwindau@mozilla.com", + "start_date": datetime.datetime(2024, 6, 10, 0, 0), + "end_date": None, + "email": ["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + "depends_on_past": False, + "retry_delay": datetime.timedelta(seconds=1800), + "email_on_failure": True, + "email_on_retry": False, + "retries": 2, +} + +tags = ["impact/tier_2", "repo/bigquery-etl"] + +with DAG( + "bqetl_census_feed", + default_args=default_args, + schedule_interval="0 17 * * *", + doc_md=docs, + tags=tags, +) as dag: + + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 = ( + ExternalTaskSensor( + task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_google_analytics_derived_ga4", + external_task_id="checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_delta=datetime.timedelta(seconds=18000), + check_existence=True, + mode="reschedule", + allowed_states=ALLOWED_STATES, + failed_states=FAILED_STATES, + pool="DATA_ENG_EXTERNALTASKSENSOR", + ) + ) + + mozilla_org_derived__ga_desktop_conversions__v1 = bigquery_etl_query( + task_id="mozilla_org_derived__ga_desktop_conversions__v1", + destination_table='ga_desktop_conversions_v1${{ macros.ds_format(macros.ds_add(ds, -2), "%Y-%m-%d", "%Y%m%d") }}', + dataset_id="mozilla_org_derived", + project_id="moz-fx-data-shared-prod", + owner="kwindau@mozilla.com", + email=["kwindau@mozilla.com", "telemetry-alerts@mozilla.com"], + date_partition_parameter=None, + depends_on_past=False, + parameters=["activity_date:DATE:{{macros.ds_add(ds, -2)}}"] + + ["submission_date:DATE:{{ds}}"], + ) + + mozilla_org_derived__ga_desktop_conversions__v1.set_upstream( + wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py --- /tmp/workspace/main-generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 21:53:03.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_google_analytics_derived_ga4.py 2024-06-06 21:53:14.000000000 +0000 @@ -153,6 +153,20 @@ retries=0, ) + with TaskGroup( + "checks__fail_mozilla_org_derived__gclid_conversions__v2_external", + ) as checks__fail_mozilla_org_derived__gclid_conversions__v2_external: + ExternalTaskMarker( + task_id="bqetl_census_feed__wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + external_dag_id="bqetl_census_feed", + external_task_id="wait_for_checks__fail_mozilla_org_derived__gclid_conversions__v2", + execution_date="{{ (execution_date - macros.timedelta(days=-1, seconds=68400)).isoformat() }}", + ) + + checks__fail_mozilla_org_derived__gclid_conversions__v2_external.set_upstream( + checks__fail_mozilla_org_derived__gclid_conversions__v2 + ) + checks__warn_ga_derived__blogs_goals__v2 = bigquery_dq_check( task_id="checks__warn_ga_derived__blogs_goals__v2", source_table="blogs_goals_v2", Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org: ga_desktop_conversions Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived: ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:50:04.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/bedrock_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:37:48.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.events_v1` + `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_live.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:50:04.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_background_tasks_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:37:49.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:50:04.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:37:49.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` UNION ALL SELECT submission_timestamp, @@ -80,7 +80,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_live.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_live.newtab_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-06 21:50:04.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/monitoring_derived/event_monitoring_aggregates_v1/query.sql 2024-06-06 21:39:25.000000000 +0000 @@ -55,7 +55,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` UNION ALL SELECT submission_timestamp, @@ -65,7 +65,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.urlbar_potential_exposure_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` UNION ALL SELECT submission_timestamp, @@ -75,7 +75,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_stable.newtab_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -582,7 +582,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -592,7 +592,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -673,7 +673,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -683,7 +683,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -764,7 +764,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -774,7 +774,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1555,7 +1555,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1565,7 +1565,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1575,7 +1575,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1646,7 +1646,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1656,7 +1656,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1666,7 +1666,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1737,7 +1737,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1747,7 +1747,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1757,7 +1757,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -1828,7 +1828,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -1838,7 +1838,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -1848,7 +1848,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_stable.main_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2000,7 +2000,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.events_v1` + `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` UNION ALL SELECT submission_timestamp, @@ -2010,7 +2010,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.bedrock_stable.non_interaction_v1` + `moz-fx-data-shared-prod.bedrock_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2081,7 +2081,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` + `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` UNION ALL SELECT submission_timestamp, @@ -2091,7 +2091,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_stable.main_events_v1` + `moz-fx-data-shared-prod.viu_politica_stable.video_index_v1` ) CROSS JOIN UNNEST(events) AS event, @@ -2162,7 +2162,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` UNION ALL SELECT submission_timestamp, @@ -2172,7 +2172,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.background_tasks_v1` + `moz-fx-data-shared-prod.firefox_desktop_background_tasks_stable.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/metadata.yaml 2024-06-06 21:50:17.000000000 +0000 @@ -0,0 +1,13 @@ +friendly_name: Ga Desktop Conversions +description: |- + Please provide a description for the query +owners: [] +labels: {} +bigquery: null +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: + view.sql: + - moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1 diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org/ga_desktop_conversions/view.sql 2024-06-06 21:36:01.000000000 +0000 @@ -0,0 +1,7 @@ +CREATE OR REPLACE VIEW + `moz-fx-data-shared-prod.mozilla_org.ga_desktop_conversions` +AS +SELECT + * +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.ga_desktop_conversions_v1` diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/metadata.yaml 2024-06-06 21:50:17.000000000 +0000 @@ -0,0 +1,30 @@ +friendly_name: Google Analytics Desktop Conversions Feed +description: |- + Query that pulls conversion event data so Census feed can pull and send to Google for marketing campaigns +owners: +- kwindau@mozilla.com +labels: + incremental: true + owner1: kwindau + dag: bqetl_census_feed +scheduling: + dag_name: bqetl_census_feed + date_partition_parameter: activity_date + date_partition_offset: -2 + parameters: + - submission_date:DATE:{{ds}} +bigquery: + time_partitioning: + type: day + field: activity_date + require_partition_filter: false + expiration_days: null + range_partitioning: null + clustering: + fields: + - conversion_name +workgroup_access: +- role: roles/bigquery.dataViewer + members: + - workgroup:mozilla-confidential +references: {} diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/query.sql 2024-06-06 21:36:01.000000000 +0000 @@ -0,0 +1,31 @@ +SELECT + activity_date, + FORMAT_DATETIME("%F %T", DATETIME(activity_date, TIME(23, 59, 59))) AS activity_datetime, + @submission_date AS run_date, + gclid, + -- Names as represented in Google Ads + -- https://docs.google.com/spreadsheets/d/1YzhhvbpOlqPLORRJUZ55BIb0H20hwFqQFApR-r0UMfI + CASE + conversion_name + WHEN "did_firefox_first_run" + THEN "firefox_first_run" + WHEN "did_search" + THEN "firefox_first_search" + WHEN "did_click_ad" + THEN "firefox_first_ad_click" + WHEN "did_returned_second_day" + THEN "firefox_second_run" + ELSE NULL + END AS conversion_name, +FROM + `moz-fx-data-shared-prod.mozilla_org_derived.gclid_conversions_v2` UNPIVOT( + did_conversion FOR conversion_name IN ( + did_firefox_first_run, + did_search, + did_click_ad, + did_returned_second_day + ) + ) +WHERE + did_conversion + AND activity_date = @activity_date diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 1970-01-01 00:00:00.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozilla_org_derived/ga_desktop_conversions_v1/schema.yaml 2024-06-06 21:36:01.000000000 +0000 @@ -0,0 +1,21 @@ +fields: +- name: activity_date + mode: NULLABLE + type: DATE + description: Activity Date +- name: activity_datetime + mode: NULLABLE + type: STRING + description: Activity Datetime +- name: run_date + mode: NULLABLE + type: DATE + description: Run Date - The submission date +- name: gclid + mode: NULLABLE + type: STRING + description: Google Click ID +- name: conversion_name + mode: NULLABLE + type: STRING + description: Conversion Name diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:50:04.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/mozillavpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:37:49.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.main_v1` + `moz-fx-data-shared-prod.mozillavpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.mozillavpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.mozillavpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:50:04.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_firefox_vpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:37:50.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_firefox_vpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:50:04.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_fennec_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:37:51.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_fennec_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:50:04.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxbeta_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:37:51.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxbeta_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:50:04.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefox_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:37:51.000000000 +0000 @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.metrics_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefox_live.events_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:50:04.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:37:51.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:50:04.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/org_mozilla_ios_firefoxvpn_network_extension_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:37:51.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.vpnsession_v1` UNION ALL SELECT submission_timestamp, @@ -70,7 +70,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.daemonsession_v1` + `moz-fx-data-shared-prod.org_mozilla_ios_firefoxvpn_network_extension_live.main_v1` ) CROSS JOIN UNNEST(events) AS event, diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml 2024-06-06 21:50:04.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/telemetry/releases_latest/schema.yaml 2024-06-06 21:45:16.000000000 +0000 @@ -2,21 +2,18 @@ - name: date type: DATE mode: NULLABLE - description: null - name: product type: STRING mode: NULLABLE - name: category type: STRING mode: NULLABLE - description: null - name: channel type: STRING mode: NULLABLE - name: build_number type: INTEGER mode: NULLABLE - description: null - name: release_date type: DATE mode: NULLABLE diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:50:04.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/viu_politica_derived/event_monitoring_live_v1/materialized_view.sql 2024-06-06 21:37:49.000000000 +0000 @@ -50,7 +50,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_live.video_index_v1` + `moz-fx-data-shared-prod.viu_politica_live.main_events_v1` UNION ALL SELECT submission_timestamp, @@ -60,7 +60,7 @@ client_info.app_display_version AS version, ping_info FROM - `moz-fx-data-shared-prod.viu_politica_live.main_events_v1` + `moz-fx-data-shared-prod.viu_politica_live.video_index_v1` ) CROSS JOIN UNNEST(events) AS event, ```

Link to full diff