updating metadata for incremental runs

[chelseybeck]

Checklist for reviewer:

• [ ] Commits should reference a bug or github issue, if relevant (if a bug is referenced, the pull request should include the bug number in the title).
• [ ] If the PR comes from a fork, trigger integration CI tests by running the Push to upstream workflow and provide the <username>:<branch> of the fork as parameter. The parameter will also show up in the logs of the manual-trigger-required-for-fork CI task together with more detailed instructions.
• [ ] If adding a new field to a query, ensure that the schema and dependent downstream schemas have been updated.
• [ ] When adding a new derived dataset, ensure that data is not available already (fully or partially) and recommend extending an existing dataset in favor of creating new ones. Data can be available in the bigquery-etl repository, looker-hub or in looker-spoke-default.

For modifications to schemas in restricted namespaces (see CODEOWNERS):

• [ ] Follow the change control procedure

┆Issue is synchronized with this Jira Task

[dataops-ci-bot]

Integration report for "updating metadata for incremental runs"

sql.diff

Click to expand!

```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_braze.py /tmp/workspace/generated-sql/dags/bqetl_braze.py --- /tmp/workspace/main-generated-sql/dags/bqetl_braze.py 2024-06-03 19:03:13.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_braze.py 2024-06-03 19:04:43.000000000 +0000 @@ -158,9 +158,8 @@ project_id="moz-fx-data-shared-prod", owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], - date_partition_parameter=None, + date_partition_parameter="updated_at", depends_on_past=False, - task_concurrency=1, ) braze_external__changed_products_sync__v1 = bigquery_etl_query( @@ -170,9 +169,8 @@ project_id="moz-fx-data-shared-prod", owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], - date_partition_parameter=None, + date_partition_parameter="updated_at", depends_on_past=False, - task_concurrency=1, ) braze_external__changed_users__v1 = bigquery_etl_query( @@ -194,9 +192,8 @@ project_id="moz-fx-data-shared-prod", owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], - date_partition_parameter=None, + date_partition_parameter="updated_at", depends_on_past=False, - task_concurrency=1, ) braze_external__changed_waitlists_sync__v1 = bigquery_etl_query( @@ -206,9 +203,8 @@ project_id="moz-fx-data-shared-prod", owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], - date_partition_parameter=None, + date_partition_parameter="updated_at", depends_on_past=False, - task_concurrency=1, ) braze_external__delete_users_sync__v1 = bigquery_etl_query( @@ -218,9 +214,8 @@ project_id="moz-fx-data-shared-prod", owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], - date_partition_parameter=None, + date_partition_parameter="updated_at", depends_on_past=False, - task_concurrency=1, ) braze_external__users_previous_day_snapshot__v1 = bigquery_etl_query( @@ -360,7 +355,7 @@ owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, - task_concurrency=1, + parameters=["updated_at:DATE:{{ds}}"], retries=0, ) @@ -373,7 +368,7 @@ owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, - task_concurrency=1, + parameters=["updated_at:DATE:{{ds}}"], retries=0, ) @@ -399,7 +394,7 @@ owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, - task_concurrency=1, + parameters=["updated_at:DATE:{{ds}}"], retries=0, ) @@ -412,7 +407,7 @@ owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, - task_concurrency=1, + parameters=["updated_at:DATE:{{ds}}"], retries=0, ) @@ -425,7 +420,7 @@ owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, - task_concurrency=1, + parameters=["updated_at:DATE:{{ds}}"], retries=0, ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_newsletters_sync_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_newsletters_sync_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_newsletters_sync_v1/metadata.yaml 2024-06-03 19:01:29.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_newsletters_sync_v1/metadata.yaml 2024-06-03 19:01:50.000000000 +0000 @@ -14,7 +14,7 @@ owner1: cbeck scheduling: dag_name: bqetl_braze - date_partition_parameter: null + date_partition_parameter: updated_at bigquery: time_partitioning: null range_partitioning: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_products_sync_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_products_sync_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_products_sync_v1/metadata.yaml 2024-06-03 19:01:29.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_products_sync_v1/metadata.yaml 2024-06-03 19:01:50.000000000 +0000 @@ -14,7 +14,7 @@ owner1: cbeck scheduling: dag_name: bqetl_braze - date_partition_parameter: null + date_partition_parameter: updated_at bigquery: time_partitioning: null range_partitioning: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/metadata.yaml 2024-06-03 19:01:29.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/metadata.yaml 2024-06-03 19:01:50.000000000 +0000 @@ -14,7 +14,7 @@ owner1: cbeck scheduling: dag_name: bqetl_braze - date_partition_parameter: null + date_partition_parameter: updated_at bigquery: time_partitioning: null range_partitioning: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_waitlists_sync_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_waitlists_sync_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_waitlists_sync_v1/metadata.yaml 2024-06-03 19:01:29.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_waitlists_sync_v1/metadata.yaml 2024-06-03 19:01:50.000000000 +0000 @@ -14,7 +14,7 @@ owner1: cbeck scheduling: dag_name: bqetl_braze - date_partition_parameter: null + date_partition_parameter: updated_at bigquery: time_partitioning: null range_partitioning: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/metadata.yaml 2024-06-03 19:01:29.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/metadata.yaml 2024-06-03 19:01:50.000000000 +0000 @@ -18,13 +18,12 @@ - cbeck@mozilla.com labels: incremental: true - schedule: daily owner: cbeck dag: bqetl_braze owner1: cbeck scheduling: dag_name: bqetl_braze - date_partition_parameter: null + date_partition_parameter: updated_at bigquery: time_partitioning: null range_partitioning: null ```

Link to full diff

[dataops-ci-bot]

Integration report for "Merge branch 'main' into 3763-fix-incremental"

sql.diff

Click to expand!

```diff diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/dags/bqetl_braze.py /tmp/workspace/generated-sql/dags/bqetl_braze.py --- /tmp/workspace/main-generated-sql/dags/bqetl_braze.py 2024-06-03 19:13:13.000000000 +0000 +++ /tmp/workspace/generated-sql/dags/bqetl_braze.py 2024-06-03 19:14:32.000000000 +0000 @@ -146,9 +146,8 @@ project_id="moz-fx-data-shared-prod", owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], - date_partition_parameter=None, + date_partition_parameter="updated_at", depends_on_past=False, - task_concurrency=1, ) braze_external__changed_newsletters_sync__v1 = bigquery_etl_query( @@ -158,9 +157,8 @@ project_id="moz-fx-data-shared-prod", owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], - date_partition_parameter=None, + date_partition_parameter="updated_at", depends_on_past=False, - task_concurrency=1, ) braze_external__changed_products_sync__v1 = bigquery_etl_query( @@ -170,9 +168,8 @@ project_id="moz-fx-data-shared-prod", owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], - date_partition_parameter=None, + date_partition_parameter="updated_at", depends_on_past=False, - task_concurrency=1, ) braze_external__changed_users__v1 = bigquery_etl_query( @@ -194,9 +191,8 @@ project_id="moz-fx-data-shared-prod", owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], - date_partition_parameter=None, + date_partition_parameter="updated_at", depends_on_past=False, - task_concurrency=1, ) braze_external__changed_waitlists_sync__v1 = bigquery_etl_query( @@ -206,9 +202,8 @@ project_id="moz-fx-data-shared-prod", owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], - date_partition_parameter=None, + date_partition_parameter="updated_at", depends_on_past=False, - task_concurrency=1, ) braze_external__delete_users_sync__v1 = bigquery_etl_query( @@ -218,9 +213,8 @@ project_id="moz-fx-data-shared-prod", owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], - date_partition_parameter=None, + date_partition_parameter="updated_at", depends_on_past=False, - task_concurrency=1, ) braze_external__users_previous_day_snapshot__v1 = bigquery_etl_query( @@ -347,7 +341,7 @@ owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, - task_concurrency=1, + parameters=["updated_at:DATE:{{ds}}"], retries=0, ) @@ -360,7 +354,7 @@ owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, - task_concurrency=1, + parameters=["updated_at:DATE:{{ds}}"], retries=0, ) @@ -373,7 +367,7 @@ owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, - task_concurrency=1, + parameters=["updated_at:DATE:{{ds}}"], retries=0, ) @@ -399,7 +393,7 @@ owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, - task_concurrency=1, + parameters=["updated_at:DATE:{{ds}}"], retries=0, ) @@ -412,7 +406,7 @@ owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, - task_concurrency=1, + parameters=["updated_at:DATE:{{ds}}"], retries=0, ) @@ -425,7 +419,7 @@ owner="cbeck@mozilla.com", email=["cbeck@mozilla.com", "leli@mozilla.com"], depends_on_past=False, - task_concurrency=1, + parameters=["updated_at:DATE:{{ds}}"], retries=0, ) diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_firefox_subscriptions_sync_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_firefox_subscriptions_sync_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_firefox_subscriptions_sync_v1/metadata.yaml 2024-06-03 19:11:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_firefox_subscriptions_sync_v1/metadata.yaml 2024-06-03 19:11:42.000000000 +0000 @@ -15,7 +15,7 @@ owner1: cbeck scheduling: dag_name: bqetl_braze - date_partition_parameter: null + date_partition_parameter: updated_at bigquery: time_partitioning: null range_partitioning: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_newsletters_sync_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_newsletters_sync_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_newsletters_sync_v1/metadata.yaml 2024-06-03 19:11:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_newsletters_sync_v1/metadata.yaml 2024-06-03 19:11:42.000000000 +0000 @@ -14,7 +14,7 @@ owner1: cbeck scheduling: dag_name: bqetl_braze - date_partition_parameter: null + date_partition_parameter: updated_at bigquery: time_partitioning: null range_partitioning: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_products_sync_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_products_sync_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_products_sync_v1/metadata.yaml 2024-06-03 19:11:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_products_sync_v1/metadata.yaml 2024-06-03 19:11:42.000000000 +0000 @@ -14,7 +14,7 @@ owner1: cbeck scheduling: dag_name: bqetl_braze - date_partition_parameter: null + date_partition_parameter: updated_at bigquery: time_partitioning: null range_partitioning: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/metadata.yaml 2024-06-03 19:11:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_users_sync_v1/metadata.yaml 2024-06-03 19:11:42.000000000 +0000 @@ -14,7 +14,7 @@ owner1: cbeck scheduling: dag_name: bqetl_braze - date_partition_parameter: null + date_partition_parameter: updated_at bigquery: time_partitioning: null range_partitioning: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_waitlists_sync_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_waitlists_sync_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_waitlists_sync_v1/metadata.yaml 2024-06-03 19:11:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/changed_waitlists_sync_v1/metadata.yaml 2024-06-03 19:11:42.000000000 +0000 @@ -14,7 +14,7 @@ owner1: cbeck scheduling: dag_name: bqetl_braze - date_partition_parameter: null + date_partition_parameter: updated_at bigquery: time_partitioning: null range_partitioning: null diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/metadata.yaml --- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/metadata.yaml 2024-06-03 19:11:26.000000000 +0000 +++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/braze_external/delete_users_sync_v1/metadata.yaml 2024-06-03 19:11:42.000000000 +0000 @@ -18,13 +18,12 @@ - cbeck@mozilla.com labels: incremental: true - schedule: daily owner: cbeck dag: bqetl_braze owner1: cbeck scheduling: dag_name: bqetl_braze - date_partition_parameter: null + date_partition_parameter: updated_at bigquery: time_partitioning: null range_partitioning: null ```

Link to full diff