feat(DENG-6890): mobile kpi support metrics managed backfill 20250122… · mozilla/bigquery-etl@73cc651

dataops-ci-bot · 2025-01-29T20:01:00Z

Integration report for "feat(DENG-6890): mobile kpi support metrics managed backfill 20250122 (#6847)"

`sql.diff`

Click to expand!

Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1: backfill.yaml
Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1: backfill.yaml
Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1: backfill.yaml
Only in /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1: backfill.yaml
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml	2025-01-29 19:38:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates/schema.yaml	2025-01-29 19:49:28.000000000 +0000
@@ -1,49 +1,49 @@
 fields:
-- mode: NULLABLE
-  name: submission_date
+- name: submission_date
   type: DATE
-- mode: NULLABLE
-  name: source
+  mode: NULLABLE
+- name: source
   type: STRING
-- mode: NULLABLE
-  name: event_type
+  mode: NULLABLE
+- name: event_type
   type: STRING
-- mode: NULLABLE
-  name: form_factor
+  mode: NULLABLE
+- name: form_factor
   type: STRING
-- mode: NULLABLE
-  name: country
+  mode: NULLABLE
+- name: country
   type: STRING
-- mode: NULLABLE
-  name: subdivision1
+  mode: NULLABLE
+- name: subdivision1
   type: STRING
-- mode: NULLABLE
-  name: advertiser
+  mode: NULLABLE
+- name: advertiser
   type: STRING
-- mode: NULLABLE
-  name: release_channel
+  mode: NULLABLE
+- name: release_channel
   type: STRING
-- mode: NULLABLE
-  name: position
+  mode: NULLABLE
+- name: position
   type: INTEGER
-- mode: NULLABLE
-  name: provider
+  mode: NULLABLE
+- name: provider
   type: STRING
-- mode: NULLABLE
-  name: match_type
+  mode: NULLABLE
+- name: match_type
   type: STRING
-- mode: NULLABLE
-  name: normalized_os
+  mode: NULLABLE
+- name: normalized_os
   type: STRING
-- mode: NULLABLE
-  name: suggest_data_sharing_enabled
+  mode: NULLABLE
+- name: suggest_data_sharing_enabled
   type: BOOLEAN
-- mode: NULLABLE
-  name: event_count
+  mode: NULLABLE
+- name: event_count
   type: INTEGER
-- mode: NULLABLE
-  name: user_count
+  mode: NULLABLE
+- name: user_count
   type: INTEGER
-- mode: NULLABLE
-  name: query_type
+  mode: NULLABLE
+- name: query_type
   type: STRING
+  mode: NULLABLE
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml	2025-01-29 19:38:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/contextual_services/event_aggregates_suggest/schema.yaml	2025-01-29 19:49:36.000000000 +0000
@@ -1,40 +1,40 @@
 fields:
-- mode: NULLABLE
-  name: submission_date
+- name: submission_date
   type: DATE
-- mode: NULLABLE
-  name: form_factor
+  mode: NULLABLE
+- name: form_factor
   type: STRING
-- mode: NULLABLE
-  name: country
+  mode: NULLABLE
+- name: country
   type: STRING
-- mode: NULLABLE
-  name: advertiser
+  mode: NULLABLE
+- name: advertiser
   type: STRING
-- mode: NULLABLE
-  name: normalized_os
+  mode: NULLABLE
+- name: normalized_os
   type: STRING
-- mode: NULLABLE
-  name: release_channel
+  mode: NULLABLE
+- name: release_channel
   type: STRING
-- mode: NULLABLE
-  name: position
+  mode: NULLABLE
+- name: position
   type: INTEGER
-- mode: NULLABLE
-  name: provider
+  mode: NULLABLE
+- name: provider
   type: STRING
-- mode: NULLABLE
-  name: match_type
+  mode: NULLABLE
+- name: match_type
   type: STRING
-- mode: NULLABLE
-  name: suggest_data_sharing_enabled
+  mode: NULLABLE
+- name: suggest_data_sharing_enabled
   type: BOOLEAN
-- mode: NULLABLE
-  name: impression_count
+  mode: NULLABLE
+- name: impression_count
   type: INTEGER
-- mode: NULLABLE
-  name: click_count
+  mode: NULLABLE
+- name: click_count
   type: INTEGER
-- mode: NULLABLE
-  name: query_type
+  mode: NULLABLE
+- name: query_type
   type: STRING
+  mode: NULLABLE
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml	2025-01-29 19:38:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_clients/schema.yaml	2025-01-29 19:49:29.000000000 +0000
@@ -26,6 +26,9 @@
 - name: adjust_network
   type: STRING
   mode: NULLABLE
+- name: install_source
+  type: STRING
+  mode: NULLABLE
 - name: retained_week_2
   type: BOOLEAN
   mode: NULLABLE
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml	2025-01-29 19:38:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/funnel_retention_week_4/schema.yaml	2025-01-29 19:49:00.000000000 +0000
@@ -48,6 +48,10 @@
   description: 'The type of source of a client installation.
 
     '
+- name: install_source
+  type: STRING
+  mode: NULLABLE
+  description: null
 - name: new_profiles
   type: INTEGER
   mode: NULLABLE
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/use_counters/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/use_counters/schema.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix/use_counters/schema.yaml	2025-01-29 19:39:09.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix/use_counters/schema.yaml	2025-01-29 19:51:10.000000000 +0000
@@ -1,11 +1,11 @@
 fields:
 - name: normalized_app_id
-  type: STRING
   mode: NULLABLE
+  type: STRING
   description: App ID of the channel data was received from
 - name: normalized_channel
-  type: STRING
   mode: NULLABLE
+  type: STRING
   description: Normalized channel name
 - name: additional_properties
   type: STRING
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1/backfill.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1/backfill.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1/backfill.yaml	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1/backfill.yaml	2025-01-29 19:39:25.000000000 +0000
@@ -0,0 +1,8 @@
+2025-01-22:
+  start_date: 2024-03-05
+  end_date: 2025-01-21
+  reason: Recompute partitions since DMA came into effect to include additional fields (DENG-6890).
+  watchers:
+  - [email protected]
+  status: Initiate
+  shredder_mitigation: false
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1/metadata.yaml	2025-01-29 19:41:27.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1/metadata.yaml	2025-01-29 19:55:40.000000000 +0000
@@ -1,13 +1,14 @@
 friendly_name: Profile / Client Engagement - Fenix (Aggregated)
 description: |-
   Profile / Client Engagement (Fenix) aggregated metrics
+
+  device_type and device_manufacturer fields were added in 2024-03-05.
 owners:
 - mozilla/kpi_table_reviewers
 - [email protected]
 labels:
   schedule: daily
   incremental: true
-  shredder_mitigation: true
   table_type: aggregate
   dag: bqetl_mobile_kpi_metrics
   owner1: kik
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/feature_usage_metrics_v2/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/feature_usage_metrics_v2/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/feature_usage_metrics_v2/metadata.yaml	2025-01-29 19:41:27.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/feature_usage_metrics_v2/metadata.yaml	2025-01-29 19:55:40.000000000 +0000
@@ -9,7 +9,6 @@
   incremental: true
   owner1: rzhao
   table_type: aggregate
-  shredder_mitigation: true
   dag: bqetl_mobile_feature_usage
   owner2: vsabino
 scheduling:
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/feature_usage_metrics_v2/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/feature_usage_metrics_v2/query.sql
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/feature_usage_metrics_v2/query.sql	2025-01-29 19:38:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/feature_usage_metrics_v2/query.sql	2025-01-29 19:39:25.000000000 +0000
@@ -1,11 +1,12 @@
 WITH baseline_clients AS (
   SELECT
     submission_date AS dau_date,
-    client_id
+    client_id,
+    LEAD(submission_date) OVER (PARTITION BY client_id ORDER BY submission_date) AS next_dau
   FROM
     `moz-fx-data-shared-prod.fenix.baseline_clients_daily`
   WHERE
-    submission_date = DATE_SUB(@submission_date, INTERVAL 4 DAY)
+    submission_date >= DATE_SUB(@submission_date, INTERVAL 4 DAY)
     AND durations > 0
     AND LOWER(COALESCE(isp, "")) <> "browserstack"
     AND LOWER(COALESCE(distribution_id, "")) <> "mozillaonline"
@@ -31,6 +32,8 @@
     AND DATE_DIFF(DATE(submission_timestamp), dau_date, DAY)
     BETWEEN 1
     AND 4
+    -- exclude metrics pings that should be matched to next DAU date
+    AND DATE(submission_timestamp) <= DATE_ADD(next_dau, INTERVAL 1 DAY)
   WHERE
     DATE(submission_timestamp)
     BETWEEN DATE_SUB(@submission_date, INTERVAL 3 DAY)
@@ -59,14 +62,34 @@
     SUM(COALESCE(metrics.counter.addresses_updated, 0)) AS addresses_modified,
     MAX(COALESCE(metrics.quantity.addresses_saved_all, 0)) AS currently_stored_addresses,
     --Bookmark
-    SUM(COALESCE(metrics_bookmarks_add_table.value, 0)) AS bookmarks_add,
-    SUM(COALESCE(metrics_bookmarks_delete_table.value, 0)) AS bookmarks_delete,
-    SUM(COALESCE(metrics_bookmarks_edit_table.value, 0)) AS bookmarks_edit,
-    SUM(COALESCE(metrics_bookmarks_open_table.value, 0)) AS bookmarks_open,
-    MAX(
+    SUM(
+      COALESCE(
+        mozfun.map.extract_keyed_scalar_sum(metrics.labeled_counter.metrics_bookmarks_add),
+        0
+      )
+    ) AS bookmarks_add,
+    SUM(
+      COALESCE(
+        mozfun.map.extract_keyed_scalar_sum(metrics.labeled_counter.metrics_bookmarks_delete),
+        0
+      )
+    ) AS bookmarks_delete,
+    SUM(
+      COALESCE(
+        mozfun.map.extract_keyed_scalar_sum(metrics.labeled_counter.metrics_bookmarks_edit),
+        0
+      )
+    ) AS bookmarks_edit,
+    SUM(
+      COALESCE(
+        mozfun.map.extract_keyed_scalar_sum(metrics.labeled_counter.metrics_bookmarks_open),
+        0
+      )
+    ) AS bookmarks_open,
+    SUM(
       COALESCE(metrics.counter.metrics_desktop_bookmarks_count, 0)
     ) AS metrics_desktop_bookmarks_count,
-    MAX(
+    SUM(
       COALESCE(metrics.counter.metrics_mobile_bookmarks_count, 0)
     ) AS metrics_mobile_bookmarks_count,
     CAST(
@@ -161,14 +184,6 @@
     ) AS customize_home_recently_visited
   FROM
     metrics_dau
-  LEFT JOIN
-    UNNEST(metrics.labeled_counter.metrics_bookmarks_add) AS metrics_bookmarks_add_table
-  LEFT JOIN
-    UNNEST(metrics.labeled_counter.metrics_bookmarks_delete) AS metrics_bookmarks_delete_table
-  LEFT JOIN
-    UNNEST(metrics.labeled_counter.metrics_bookmarks_edit) AS metrics_bookmarks_edit_table
-  LEFT JOIN
-    UNNEST(metrics.labeled_counter.metrics_bookmarks_open) AS metrics_bookmarks_open_table
   GROUP BY
     dau_date,
     client_id
@@ -322,6 +337,8 @@
 LEFT JOIN
   client_attribution
   USING (client_id)
+WHERE
+  dau_date = DATE_SUB(@submission_date, INTERVAL 4 DAY)
 GROUP BY
   submission_date,
   metric_date,
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1/backfill.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1/backfill.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1/backfill.yaml	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1/backfill.yaml	2025-01-29 19:39:25.000000000 +0000
@@ -0,0 +1,8 @@
+2025-01-22:
+  start_date: 2024-03-05
+  end_date: 2025-01-21
+  reason: Recompute partitions since DMA came into effect to include additional fields (DENG-6890).
+  watchers:
+  - [email protected]
+  status: Initiate
+  shredder_mitigation: false
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1/metadata.yaml	2025-01-29 19:41:27.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1/metadata.yaml	2025-01-29 19:55:40.000000000 +0000
@@ -1,13 +1,14 @@
 friendly_name: New profiles - Fenix (Aggregated)
 description: |-
   New profiles (Fenix) aggregated metrics
+
+  device_type and device_manufacturer fields were added in 2024-03-05.
 owners:
 - mozilla/kpi_table_reviewers
 - [email protected]
 labels:
   schedule: daily
   incremental: true
-  shredder_mitigation: true
   table_type: aggregate
   dag: bqetl_mobile_kpi_metrics
   owner1: kik
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/backfill.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/backfill.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/backfill.yaml	2025-01-29 19:38:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/backfill.yaml	2025-01-29 19:39:25.000000000 +0000
@@ -1,3 +1,12 @@
+2025-01-22:
+  start_date: 2024-03-05
+  end_date: 2025-01-21
+  reason: Recompute partitions since DMA came into effect to include additional fields (DENG-6890).
+  watchers:
+  - [email protected]
+  status: Initiate
+  shredder_mitigation: false
+
 2024-05-31:
   start_date: 2021-01-01
   end_date: 2024-05-31
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/metadata.yaml	2025-01-29 19:41:27.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/metadata.yaml	2025-01-29 19:55:40.000000000 +0000
@@ -6,13 +6,14 @@
   Client's attribute values as observed on the metric date.
 
   see: DENG-3183 for more information.
+
+  device_type and device_manufacturer fields were added in 2024-03-05.
 owners:
 - mozilla/kpi_table_reviewers
 - [email protected]
 labels:
   schedule: daily
   incremental: true
-  shredder_mitigation: true
   table_type: aggregate
   dag: bqetl_mobile_kpi_metrics
   owner1: kik
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/ltv_states/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/ltv_states/schema.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/ltv_states/schema.yaml	2025-01-29 19:38:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/ltv_states/schema.yaml	2025-01-29 19:48:54.000000000 +0000
@@ -1,64 +1,66 @@
 fields:
-- description: Unique ID for the client installation.
-  mode: NULLABLE
-  name: client_id
+- name: client_id
   type: STRING
-- description: Sample ID - A number ranging from 0 - 99 based on client ID; used to pull a small sample of data related to a subset of clients over time
   mode: NULLABLE
-  name: sample_id
-  type: INT64
-- description: Submission Date
+  description: Unique ID for the client installation.
+- name: sample_id
+  type: INTEGER
   mode: NULLABLE
-  name: submission_date
+  description: Sample ID - A number ranging from 0 - 99 based on client ID; used to
+    pull a small sample of data related to a subset of clients over time
+- name: submission_date
   type: DATE
-- description: First Seen Date - The date this client was first seen
   mode: NULLABLE
-  name: first_seen_date
+  description: Submission Date
+- name: first_seen_date
   type: DATE
-- description: Days Since First Seen - The number of days since the client was first seen
   mode: NULLABLE
-  name: days_since_first_seen
-  type: INT64
-- description: Days Since Active
+  description: First Seen Date - The date this client was first seen
+- name: days_since_first_seen
+  type: INTEGER
   mode: NULLABLE
-  name: days_since_active
-  type: INT64
-- description: First Reported Country - The country this client ID was first reported from
+  description: Days Since First Seen - The number of days since the client was first
+    seen
+- name: days_since_active
+  type: INTEGER
   mode: NULLABLE
-  name: first_reported_country
+  description: Days Since Active
+- name: first_reported_country
   type: STRING
-- description: Attribution
   mode: NULLABLE
-  name: attribution
+  description: First Reported Country - The country this client ID was first reported
+    from
+- name: attribution
   type: RECORD
+  mode: NULLABLE
   fields:
-  - mode: NULLABLE
-    name: source
+  - name: source
     type: STRING
+    mode: NULLABLE
     description: Attribution Source
-  - mode: NULLABLE
-    name: medium
+  - name: medium
     type: STRING
+    mode: NULLABLE
     description: Attribution Medium
-  - mode: NULLABLE
-    name: campaign
+  - name: campaign
     type: STRING
+    mode: NULLABLE
     description: Attribution Campaign
-  - mode: NULLABLE
-    name: content
+  - name: content
     type: STRING
+    mode: NULLABLE
     description: Attribution Content
-  - mode: NULLABLE
-    name: experiment
+  - name: experiment
     type: STRING
+    mode: NULLABLE
     description: Attribution Experiment
-  - mode: NULLABLE
-    name: variation
+  - name: variation
     type: STRING
+    mode: NULLABLE
     description: Attribution Variation
-  - mode: NULLABLE
-    name: dltoken
+  - name: dltoken
     type: STRING
+    mode: NULLABLE
     description: Attribution Download Token
   - name: dlsource
     type: STRING
@@ -68,40 +70,43 @@
     type: STRING
     mode: NULLABLE
     description: Attribution UA
-- description: Active
-  mode: NULLABLE
-  name: active
-  type: INT64
-- description: Ad Clicks - The number of ad clicks from this client on the submission date
+  description: Attribution
+- name: active
+  type: INTEGER
   mode: NULLABLE
-  name: ad_clicks
-  type: INT64
-- description: Total Historic Ad Clicks - The number of ad clicks from this client on or before the submission date
+  description: Active
+- name: ad_clicks
+  type: INTEGER
   mode: NULLABLE
-  name: total_historic_ad_clicks
-  type: INT64
-- description: Days Seen Bytes
+  description: Ad Clicks - The number of ad clicks from this client on the submission
+    date
+- name: total_historic_ad_clicks
+  type: INTEGER
   mode: NULLABLE
-  name: days_seen_bytes
+  description: Total Historic Ad Clicks - The number of ad clicks from this client
+    on or before the submission date
+- name: days_seen_bytes
   type: BYTES
-- description: Pattern
   mode: NULLABLE
-  name: pattern
+  description: Days Seen Bytes
+- name: pattern
   type: INTEGER
-- description: Death Time
   mode: NULLABLE
-  name: death_time
+  description: Pattern
+- name: death_time
   type: INTEGER
-- description: Max Days
   mode: NULLABLE
-  name: max_days
+  description: Death Time
+- name: max_days
   type: INTEGER
-- description: Markov States
   mode: NULLABLE
-  name: markov_states
+  description: Max Days
+- name: markov_states
   type: RECORD
-  fields:
-  - description: Desktop States V1
     mode: NULLABLE
-    name: desktop_states_v1
+  fields:
+  - name: desktop_states_v1
     type: STRING
+    mode: NULLABLE
+    description: Desktop States V1
+  description: Markov States
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/newtab_live/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/newtab_live/schema.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/newtab_live/schema.yaml	2025-01-29 19:38:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_desktop/newtab_live/schema.yaml	2025-01-29 19:49:02.000000000 +0000
@@ -1,48 +1,47 @@
 fields:
-- description: Submission Timestamp
-  mode: NULLABLE
-  name: submission_timestamp
+- name: submission_timestamp
   type: TIMESTAMP
-- description: Normalized Country Code, Examples - US, AR, BR, etc.
   mode: NULLABLE
-  name: normalized_country_code
+  description: Submission Timestamp
+- name: normalized_country_code
   type: STRING
-- description: Normalized Channel, Examples - release, nightly, aurora, esr, beta
   mode: NULLABLE
-  name: normalized_channel
+  description: Normalized Country Code, Examples - US, AR, BR, etc.
+- name: normalized_channel
   type: STRING
-- description: Document ID
   mode: NULLABLE
-  name: document_id
+  description: Normalized Channel, Examples - release, nightly, aurora, esr, beta
+- name: document_id
   type: STRING
-- description: Pocket Enabled
   mode: NULLABLE
-  name: pocket_enabled
+  description: Document ID
+- name: pocket_enabled
   type: BOOLEAN
-- description: Pocket Sponsored Stories Enabled
   mode: NULLABLE
-  name: pocket_sponsored_stories_enabled
+  description: Pocket Enabled
+- name: pocket_sponsored_stories_enabled
   type: BOOLEAN
-- description: Newtab Locale
   mode: NULLABLE
-  name: newtab_locale
+  description: Pocket Sponsored Stories Enabled
+- name: newtab_locale
   type: STRING
-- description: App Build
   mode: NULLABLE
-  name: app_build
+  description: Newtab Locale
+- name: app_build
   type: STRING
-- description: App Display Version
   mode: NULLABLE
-  name: app_display_version
+  description: App Build
+- name: app_display_version
   type: STRING
-- description: Client ID
   mode: NULLABLE
-  name: client_id
+  description: App Display Version
+- name: client_id
   type: STRING
+  mode: NULLABLE
+  description: Client ID
 - name: events
   type: RECORD
   mode: REPEATED
-  description: Events
   fields:
   - name: category
     type: STRING
@@ -51,7 +50,6 @@
   - name: extra
     type: RECORD
     mode: REPEATED
-    description: Extras
     fields:
     - name: key
       type: STRING
@@ -61,6 +59,7 @@
       type: STRING
       mode: NULLABLE
       description: Value
+    description: Extras
   - name: name
     type: STRING
     mode: NULLABLE
@@ -69,3 +68,4 @@
     type: INTEGER
     mode: NULLABLE
     description: Event Timestamp
+  description: Events
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1/backfill.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1/backfill.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1/backfill.yaml	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1/backfill.yaml	2025-01-29 19:39:25.000000000 +0000
@@ -0,0 +1,8 @@
+2025-01-22:
+  start_date: 2024-03-05
+  end_date: 2025-01-21
+  reason: Recompute partitions since DMA came into effect to include additional fields (DENG-6890).
+  watchers:
+  - [email protected]
+  status: Initiate
+  shredder_mitigation: false
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1/metadata.yaml	2025-01-29 19:41:21.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1/metadata.yaml	2025-01-29 19:55:37.000000000 +0000
@@ -1,13 +1,14 @@
 friendly_name: Profile / Client Engagement - Firefox iOS (Aggregated)
 description: |-
   Profile / Client Engagement (Firefox iOS) aggregated metrics
+
+  device_type and device_manufacturer fields were added in 2024-03-05.
 owners:
 - mozilla/kpi_table_reviewers
 - [email protected]
 labels:
   schedule: daily
   incremental: true
-  shredder_mitigation: true
   table_type: aggregate
   dag: bqetl_mobile_kpi_metrics
   owner1: kik
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/feature_usage_metrics_v2/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/feature_usage_metrics_v2/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/feature_usage_metrics_v2/metadata.yaml	2025-01-29 19:41:21.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/feature_usage_metrics_v2/metadata.yaml	2025-01-29 19:55:37.000000000 +0000
@@ -9,7 +9,6 @@
   incremental: true
   owner1: rzhao
   table_type: aggregate
-  shredder_mitigation: true
   dag: bqetl_mobile_feature_usage
   owner2: vsabino
 scheduling:
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/feature_usage_metrics_v2/query.sql /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/feature_usage_metrics_v2/query.sql
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/feature_usage_metrics_v2/query.sql	2025-01-29 19:38:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/feature_usage_metrics_v2/query.sql	2025-01-29 19:39:25.000000000 +0000
@@ -1,11 +1,12 @@
 WITH baseline_clients AS (
   SELECT
     submission_date AS dau_date,
-    client_id
+    client_id,
+    LEAD(submission_date) OVER (PARTITION BY client_id ORDER BY submission_date) AS next_dau
   FROM
     `moz-fx-data-shared-prod.firefox_ios.baseline_clients_daily`
   WHERE
-    submission_date = DATE_SUB(@submission_date, INTERVAL 4 DAY)
+    submission_date >= DATE_SUB(@submission_date, INTERVAL 4 DAY)
     AND durations > 0
     AND LOWER(COALESCE(isp, "")) <> "browserstack"
 ),
@@ -30,6 +31,8 @@
     AND DATE_DIFF(DATE(submission_timestamp), dau_date, DAY)
     BETWEEN 1
     AND 4
+    -- exclude metrics pings that should be matched to next DAU date
+    AND DATE(submission_timestamp) <= DATE_ADD(next_dau, INTERVAL 1 DAY)
   WHERE
     DATE(submission_timestamp)
     BETWEEN DATE_SUB(@submission_date, INTERVAL 3 DAY)
@@ -63,15 +66,25 @@
     SUM(COALESCE(metrics.counter.credit_card_saved, 0)) AS credit_card_saved,
     MAX(COALESCE(metrics.quantity.credit_card_saved_all, 0)) AS credit_card_saved_all,
     --Bookmark
-    SUM(COALESCE(bookmarks_add_table.value, 0)) AS bookmarks_add,
-    SUM(COALESCE(bookmarks_delete_table.value, 0)) AS bookmarks_delete,
-    SUM(COALESCE(bookmarks_edit_table.value, 0)) AS bookmarks_edit,
+    SUM(
+      COALESCE(mozfun.map.extract_keyed_scalar_sum(metrics.labeled_counter.bookmarks_add), 0)
+    ) AS bookmarks_add,
+    SUM(
+      COALESCE(mozfun.map.extract_keyed_scalar_sum(metrics.labeled_counter.bookmarks_delete), 0)
+    ) AS bookmarks_delete,
+    SUM(
+      COALESCE(mozfun.map.extract_keyed_scalar_sum(metrics.labeled_counter.bookmarks_edit), 0)
+    ) AS bookmarks_edit,
+    SUM(
+      COALESCE(mozfun.map.extract_keyed_scalar_sum(metrics.labeled_counter.bookmarks_open), 0)
+    ) AS bookmarks_open,
+    SUM(
+      COALESCE(mozfun.map.extract_keyed_scalar_sum(metrics.labeled_counter.bookmarks_view_list), 0)
+    ) AS bookmarks_view_list,
     CAST(
       MAX(COALESCE(metrics.boolean.bookmarks_has_mobile_bookmarks, FALSE)) AS INT64
     ) AS has_mobile_bookmarks,
     MAX(COALESCE(metrics.quantity.bookmarks_mobile_bookmarks_count, 0)) AS mobile_bookmarks_count,
-    SUM(COALESCE(bookmarks_open_table.value, 0)) AS bookmarks_open,
-    SUM(COALESCE(bookmarks_view_list_table.value, 0)) AS bookmarks_view_list,
     --FxA
     SUM(COALESCE(metrics.counter.sync_create_account_pressed, 0)) AS sync_create_account_pressed,
     SUM(COALESCE(metrics.counter.sync_open_tab, 0)) AS sync_open_tab,
@@ -123,16 +136,6 @@
     MAX(COALESCE(metrics.quantity.addresses_saved_all, 0)) AS addresses_saved_all
   FROM
     metrics_dau
-  LEFT JOIN
-    UNNEST(metrics.labeled_counter.bookmarks_add) AS bookmarks_add_table
-  LEFT JOIN
-    UNNEST(metrics.labeled_counter.bookmarks_delete) AS bookmarks_delete_table
-  LEFT JOIN
-    UNNEST(metrics.labeled_counter.bookmarks_edit) AS bookmarks_edit_table
-  LEFT JOIN
-    UNNEST(metrics.labeled_counter.bookmarks_open) AS bookmarks_open_table
-  LEFT JOIN
-    UNNEST(metrics.labeled_counter.bookmarks_view_list) AS bookmarks_view_list_table
   GROUP BY
     dau_date,
     client_id
@@ -307,6 +310,8 @@
 LEFT JOIN
   client_attribution
   USING (client_id)
+WHERE
+  dau_date = DATE_SUB(@submission_date, INTERVAL 4 DAY)
 GROUP BY
   submission_date,
   metric_date,
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1/backfill.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1/backfill.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1/backfill.yaml	1970-01-01 00:00:00.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1/backfill.yaml	2025-01-29 19:39:25.000000000 +0000
@@ -0,0 +1,8 @@
+2025-01-22:
+  start_date: 2024-03-05
+  end_date: 2025-01-21
+  reason: Recompute partitions since DMA came into effect to include additional fields (DENG-6890).
+  watchers:
+  - [email protected]
+  status: Initiate
+  shredder_mitigation: false
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1/metadata.yaml	2025-01-29 19:41:21.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1/metadata.yaml	2025-01-29 19:55:37.000000000 +0000
@@ -1,13 +1,14 @@
 friendly_name: New profiles - Firefox iOS (Aggregated)
 description: |-
   New profiles (Firefox iOS) aggregated metrics
+
+  device_type and device_manufacturer fields were added in 2024-03-05.
 owners:
 - mozilla/kpi_table_reviewers
 - [email protected]
 labels:
   schedule: daily
   incremental: true
-  shredder_mitigation: true
   table_type: aggregate
   dag: bqetl_mobile_kpi_metrics
   owner1: kik
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/retention_v1/backfill.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/retention_v1/backfill.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/retention_v1/backfill.yaml	2025-01-29 19:38:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/retention_v1/backfill.yaml	2025-01-29 19:39:25.000000000 +0000
@@ -1,3 +1,12 @@
+2025-01-22:
+  start_date: 2024-03-05
+  end_date: 2025-01-21
+  reason: Recompute partitions since DMA came into effect to include additional fields (DENG-6890).
+  watchers:
+  - [email protected]
+  status: Initiate
+  shredder_mitigation: false
+
 2024-06-04:
   start_date: 2021-01-01
   end_date: 2024-06-04
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/retention_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/retention_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/retention_v1/metadata.yaml	2025-01-29 19:41:21.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/firefox_ios_derived/retention_v1/metadata.yaml	2025-01-29 19:55:37.000000000 +0000
@@ -6,13 +6,14 @@
   Client's attribute values as observed on the metric date.
 
   see: DENG-3183 for more information.
+
+  device_type and device_manufacturer fields were added in 2024-03-05.
 owners:
 - mozilla/kpi_table_reviewers
 - [email protected]
 labels:
   schedule: daily
   incremental: true
-  shredder_mitigation: true
   table_type: aggregate
   dag: bqetl_mobile_kpi_metrics
   owner1: kik
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/focus_android_derived/engagement_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/focus_android_derived/engagement_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/focus_android_derived/engagement_v1/metadata.yaml	2025-01-29 19:41:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/focus_android_derived/engagement_v1/metadata.yaml	2025-01-29 19:55:41.000000000 +0000
@@ -1,13 +1,14 @@
 friendly_name: Profile / Client Engagement - Focus Android (Aggregated)
 description: |-
   Profile / Client Engagement (Focus Android) aggregated metrics
+
+  device_type and device_manufacturer fields were added on 2025-01-18.
 owners:
 - mozilla/kpi_table_reviewers
 - [email protected]
 labels:
   schedule: daily
   incremental: true
-  shredder_mitigation: true
   table_type: aggregate
   dag: bqetl_mobile_kpi_metrics
   owner1: kik
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/focus_android_derived/new_profiles_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/focus_android_derived/new_profiles_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/focus_android_derived/new_profiles_v1/metadata.yaml	2025-01-29 19:41:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/focus_android_derived/new_profiles_v1/metadata.yaml	2025-01-29 19:55:41.000000000 +0000
@@ -1,13 +1,14 @@
 friendly_name: New profiles - Focus Android (Aggregated)
 description: |-
   New profiles (Focus Android) aggregated metrics
+
+  device_type and device_manufacturer fields were added on 2025-01-18.
 owners:
 - mozilla/kpi_table_reviewers
 - [email protected]
 labels:
   schedule: daily
   incremental: true
-  shredder_mitigation: true
   table_type: aggregate
   dag: bqetl_mobile_kpi_metrics
   owner1: kik
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/focus_android_derived/retention_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/focus_android_derived/retention_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/focus_android_derived/retention_v1/metadata.yaml	2025-01-29 19:41:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/focus_android_derived/retention_v1/metadata.yaml	2025-01-29 19:55:41.000000000 +0000
@@ -6,13 +6,14 @@
   Client's attribute values as observed on the metric date.
 
   see: DENG-3183 for more information.
+
+  device_type and device_manufacturer fields were added on 2025-01-18.
 owners:
 - mozilla/kpi_table_reviewers
 - [email protected]
 labels:
   schedule: daily
   incremental: true
-  shredder_mitigation: true
   table_type: aggregate
   dag: bqetl_mobile_kpi_metrics
   owner1: kik
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/focus_ios_derived/engagement_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/focus_ios_derived/engagement_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/focus_ios_derived/engagement_v1/metadata.yaml	2025-01-29 19:41:21.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/focus_ios_derived/engagement_v1/metadata.yaml	2025-01-29 19:55:37.000000000 +0000
@@ -1,13 +1,14 @@
 friendly_name: Profile / Client Engagement - Focus iOS (Aggregated)
 description: |-
   Profile / Client Engagement (Focus iOS) aggregated metrics
+
+  device_type and device_manufacturer fields were added on 2025-01-18.
 owners:
 - mozilla/kpi_table_reviewers
 - [email protected]
 labels:
   schedule: daily
   incremental: true
-  shredder_mitigation: true
   table_type: aggregate
   dag: bqetl_mobile_kpi_metrics
   owner1: kik
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/focus_ios_derived/new_profiles_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/focus_ios_derived/new_profiles_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/focus_ios_derived/new_profiles_v1/metadata.yaml	2025-01-29 19:41:21.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/focus_ios_derived/new_profiles_v1/metadata.yaml	2025-01-29 19:55:37.000000000 +0000
@@ -1,13 +1,14 @@
 friendly_name: New profiles - Focus iOS (Aggregated)
 description: |-
   New profiles (Focus iOS) aggregated metrics
+
+  device_type and device_manufacturer fields were added on 2025-01-18.
 owners:
 - mozilla/kpi_table_reviewers
 - [email protected]
 labels:
   schedule: daily
   incremental: true
-  shredder_mitigation: true
   table_type: aggregate
   dag: bqetl_mobile_kpi_metrics
   owner1: kik
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/focus_ios_derived/retention_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/focus_ios_derived/retention_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/focus_ios_derived/retention_v1/metadata.yaml	2025-01-29 19:41:21.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/focus_ios_derived/retention_v1/metadata.yaml	2025-01-29 19:55:37.000000000 +0000
@@ -6,13 +6,14 @@
   Client's attribute values as observed on the metric date.
 
   see: DENG-3183 for more information.
+
+  device_type and device_manufacturer fields were added on 2025-01-18.
 owners:
 - mozilla/kpi_table_reviewers
 - [email protected]
 labels:
   schedule: daily
   incremental: true
-  shredder_mitigation: true
   table_type: aggregate
   dag: bqetl_mobile_kpi_metrics
   owner1: kik
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/google_search_console/limited_historical_search_impressions_by_page/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/google_search_console/limited_historical_search_impressions_by_page/schema.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/google_search_console/limited_historical_search_impressions_by_page/schema.yaml	2025-01-29 19:38:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/google_search_console/limited_historical_search_impressions_by_page/schema.yaml	2025-01-29 19:49:06.000000000 +0000
@@ -6,9 +6,10 @@
 - name: site_url
   type: STRING
   mode: NULLABLE
-  description: |-
-    For domain properties, this will be `sc-domain:` followed by the domain name.
-    For URL-prefix properties, it will be the full URL of the property definition.
+  description: 'For domain properties, this will be `sc-domain:` followed by the domain
+    name.
+
+    For URL-prefix properties, it will be the full URL of the property definition.'
 - name: site_domain_name
   type: STRING
   mode: NULLABLE
@@ -28,19 +29,23 @@
 - name: localized_site_code
   type: STRING
   mode: NULLABLE
-  description: Localized site code such as `en-US` or `de` found in the first segment of the page URL path (if any).
+  description: Localized site code such as `en-US` or `de` found in the first segment
+    of the page URL path (if any).
 - name: localized_site
   type: STRING
   mode: NULLABLE
-  description: Description of the localized site language and/or country based on `localized_site_code` (if any).
+  description: Description of the localized site language and/or country based on
+    `localized_site_code` (if any).
 - name: localized_site_language_code
   type: STRING
   mode: NULLABLE
-  description: Localized site language code in ISO-639-alpha-2 format found in the first segment of the page URL path (if any).
+  description: Localized site language code in ISO-639-alpha-2 format found in the
+    first segment of the page URL path (if any).
 - name: localized_site_language
   type: STRING
   mode: NULLABLE
-  description: Localized site language based on `localized_site_language_code` (if any).
+  description: Localized site language based on `localized_site_language_code` (if
+    any).
 - name: query
   type: STRING
   mode: NULLABLE
@@ -48,20 +53,15 @@
 - name: query_type
   type: STRING
   mode: NULLABLE
-  description: |-
-    Type of search query:
-      * Brand: Query contained one or more Mozilla brand keywords.
-      * Non-Brand: Query didn't contain any Mozilla brand keywords.
-      * Unknown: Query couldn't be classified.
+  description: "Type of search query:\n  * Brand: Query contained one or more Mozilla\
+    \ brand keywords.\n  * Non-Brand: Query didn't contain any Mozilla brand keywords.\n\
+    \  * Unknown: Query couldn't be classified."
 - name: search_type
   type: STRING
   mode: NULLABLE
-  description: |-
-    Where the link was seen by the user:
-      * Web: In Google Search's default "All" tab.
-      * Image: In Google Search's "Images" tab.
-      * Video: In Google Search's "Videos" tab.
-      * News: In Google Search's "News" tab.
+  description: "Where the link was seen by the user:\n  * Web: In Google Search's\
+    \ default \"All\" tab.\n  * Image: In Google Search's \"Images\" tab.\n  * Video:\
+    \ In Google Search's \"Videos\" tab.\n  * News: In Google Search's \"News\" tab."
 - name: user_country_code
   type: STRING
   mode: NULLABLE
@@ -81,12 +81,13 @@
 - name: device_type
   type: STRING
   mode: NULLABLE
-  description: |-
-    The type of device on which the user was searching: Desktop, Mobile, or Tablet.
+  description: 'The type of device on which the user was searching: Desktop, Mobile,
+    or Tablet.'
 - name: impressions
   type: INTEGER
   mode: NULLABLE
-  description: The number of times that search results with a link to the page were shown to a user.
+  description: The number of times that search results with a link to the page were
+    shown to a user.
 - name: clicks
   type: INTEGER
   mode: NULLABLE
@@ -94,4 +95,5 @@
 - name: average_position
   type: FLOAT
   mode: NULLABLE
-  description: The average position of the page in the search results, where `1` is the topmost position.
+  description: The average position of the page in the search results, where `1` is
+    the topmost position.
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/google_search_console/limited_historical_search_impressions_by_site/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/google_search_console/limited_historical_search_impressions_by_site/schema.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/google_search_console/limited_historical_search_impressions_by_site/schema.yaml	2025-01-29 19:38:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/google_search_console/limited_historical_search_impressions_by_site/schema.yaml	2025-01-29 19:49:25.000000000 +0000
@@ -6,9 +6,10 @@
 - name: site_url
   type: STRING
   mode: NULLABLE
-  description: |-
-    For domain properties, this will be `sc-domain:` followed by the domain name.
-    For URL-prefix properties, it will be the full URL of the property definition.
+  description: 'For domain properties, this will be `sc-domain:` followed by the domain
+    name.
+
+    For URL-prefix properties, it will be the full URL of the property definition.'
 - name: site_domain_name
   type: STRING
   mode: NULLABLE
@@ -20,20 +21,15 @@
 - name: query_type
   type: STRING
   mode: NULLABLE
-  description: |-
-    Type of search query:
-      * Brand: Query contained one or more Mozilla brand keywords.
-      * Non-Brand: Query didn't contain any Mozilla brand keywords.
-      * Unknown: Query couldn't be classified.
+  description: "Type of search query:\n  * Brand: Query contained one or more Mozilla\
+    \ brand keywords.\n  * Non-Brand: Query didn't contain any Mozilla brand keywords.\n\
+    \  * Unknown: Query couldn't be classified."
 - name: search_type
   type: STRING
   mode: NULLABLE
-  description: |-
-    Where the link was seen by the user:
-      * Web: In Google Search's default "All" tab.
-      * Image: In Google Search's "Images" tab.
-      * Video: In Google Search's "Videos" tab.
-      * News: In Google Search's "News" tab.
+  description: "Where the link was seen by the user:\n  * Web: In Google Search's\
+    \ default \"All\" tab.\n  * Image: In Google Search's \"Images\" tab.\n  * Video:\
+    \ In Google Search's \"Videos\" tab.\n  * News: In Google Search's \"News\" tab."
 - name: user_country_code
   type: STRING
   mode: NULLABLE
@@ -53,17 +49,20 @@
 - name: device_type
   type: STRING
   mode: NULLABLE
-  description: |-
-    The type of device on which the user was searching: Desktop, Mobile, or Tablet.
+  description: 'The type of device on which the user was searching: Desktop, Mobile,
+    or Tablet.'
 - name: impressions
   type: INTEGER
   mode: NULLABLE
-  description: The number of times that search results with at least one link to the site were shown to a user.
+  description: The number of times that search results with at least one link to the
+    site were shown to a user.
 - name: clicks
   type: INTEGER
   mode: NULLABLE
-  description: The number of times a user clicked at least one search result link to the site.
+  description: The number of times a user clicked at least one search result link
+    to the site.
 - name: average_top_position
   type: FLOAT
   mode: NULLABLE
-  description: The average top position of the site in the search results, where `1` is the topmost position.
+  description: The average top position of the site in the search results, where `1`
+    is the topmost position.
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/google_search_console/search_impressions_by_page/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/google_search_console/search_impressions_by_page/schema.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/google_search_console/search_impressions_by_page/schema.yaml	2025-01-29 19:38:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/google_search_console/search_impressions_by_page/schema.yaml	2025-01-29 19:49:05.000000000 +0000
@@ -6,9 +6,10 @@
 - name: site_url
   type: STRING
   mode: NULLABLE
-  description: |-
-    For domain properties, this will be `sc-domain:` followed by the domain name.
-    For URL-prefix properties, it will be the full URL of the property definition.
+  description: 'For domain properties, this will be `sc-domain:` followed by the domain
+    name.
+
+    For URL-prefix properties, it will be the full URL of the property definition.'
 - name: site_domain_name
   type: STRING
   mode: NULLABLE
@@ -16,99 +17,104 @@
 - name: page_url
   type: STRING
   mode: NULLABLE
-  description: |-
-    The final page URL linked by a search result after any skip redirects.
-    This will be null for anonymized Discover impressions.
+  description: 'The final page URL linked by a search result after any skip redirects.
+
+    This will be null for anonymized Discover impressions.'
 - name: page_domain_name
   type: STRING
   mode: NULLABLE
-  description: |-
-    Domain name of the page URL.
-    This will be null for anonymized Discover impressions.
+  description: 'Domain name of the page URL.
+
+    This will be null for anonymized Discover impressions.'
 - name: page_path
   type: STRING
   mode: NULLABLE
-  description: |-
-    The path part of the page URL.
-    This will be null for anonymized Discover impressions.
+  description: 'The path part of the page URL.
+
+    This will be null for anonymized Discover impressions.'
 - name: localized_site_code
   type: STRING
   mode: NULLABLE
-  description: |-
-    Localized site code such as `en-US` or `de` found in the first segment of the page URL path (if any).
-    This will be null for anonymized Discover impressions.
+  description: 'Localized site code such as `en-US` or `de` found in the first segment
+    of the page URL path (if any).
+
+    This will be null for anonymized Discover impressions.'
 - name: localized_site
   type: STRING
   mode: NULLABLE
-  description: |-
-    Description of the localized site language and/or country based on `localized_site_code` (if any).
-    This will be null for anonymized Discover impressions.
+  description: 'Description of the localized site language and/or country based on
+    `localized_site_code` (if any).
+
+    This will be null for anonymized Discover impressions.'
 - name: localized_site_language_code
   type: STRING
   mode: NULLABLE
-  description: |-
-    Localized site language code in ISO-639-alpha-2 format found in the first segment of the page URL path (if any).
-    This will be null for anonymized Discover impressions.
+  description: 'Localized site language code in ISO-639-alpha-2 format found in the
+    first segment of the page URL path (if any).
+
+    This will be null for anonymized Discover impressions.'
 - name: localized_site_language
   type: STRING
   mode: NULLABLE
-  description: |-
-    Localized site language based on `localized_site_language_code` (if any).
-    This will be null for anonymized Discover impressions.
+  description: 'Localized site language based on `localized_site_language_code` (if
+    any).
+
+    This will be null for anonymized Discover impressions.'
 - name: query
   type: STRING
   mode: NULLABLE
-  description: |-
-    The search query.
-    This will be null for anonymized search impressions, and all Discover and Google News search impressions.
+  description: 'The search query.
+
+    This will be null for anonymized search impressions, and all Discover and Google
+    News search impressions.'
 - name: query_type
   type: STRING
   mode: NULLABLE
-  description: |-
-    Type of search query:
-      * Anonymized: Query was redacted by Google to protect the users' privacy.
-      * Brand: Query contained one or more Mozilla brand keywords.
-      * Non-Brand: Query didn't contain any Mozilla brand keywords.
-      * Unknown: Query couldn't be classified.
-    This will be null for all Discover and Google News search impressions.
+  description: "Type of search query:\n  * Anonymized: Query was redacted by Google\
+    \ to protect the users' privacy.\n  * Brand: Query contained one or more Mozilla\
+    \ brand keywords.\n  * Non-Brand: Query didn't contain any Mozilla brand keywords.\n\
+    \  * Unknown: Query couldn't be classified.\nThis will be null for all Discover\
+    \ and Google News search impressions."
 - name: is_anonymized
   type: BOOLEAN
   mode: NULLABLE
-  description: |-
-    Whether Google has anonymized the search impression to protect the users' privacy.
+  description: 'Whether Google has anonymized the search impression to protect the
+    users'' privacy.
+
     The `query` field will be null for anonymized search impressions.
-    The `country_code`, `page_url`, and related fields will be null for anonymized Discover impressions.
+
+    The `country_code`, `page_url`, and related fields will be null for anonymized
+    Discover impressions.'
 - name: has_good_page_experience
   type: BOOLEAN
   mode: NULLABLE
-  description: Whether Google Search considers the page to be providing a good page experience.
+  description: Whether Google Search considers the page to be providing a good page
+    experience.
 - name: search_type
   type: STRING
   mode: NULLABLE
-  description: |-
-    Where the link was seen by the user:
-      * Web: In Google Search's default "All" tab.
-      * Image: In Google Search's "Images" tab.
-      * Video: In Google Search's "Videos" tab.
-      * News: In Google Search's "News" tab.
-      * Discover: In Google's Discover feed.
-      * Google News: On news.google.com or in the Google News app on Android and iOS.
+  description: "Where the link was seen by the user:\n  * Web: In Google Search's\
+    \ default \"All\" tab.\n  * Image: In Google Search's \"Images\" tab.\n  * Video:\
+    \ In Google Search's \"Videos\" tab.\n  * News: In Google Search's \"News\" tab.\n\
+    \  * Discover: In Google's Discover feed.\n  * Google News: On news.google.com\
+    \ or in the Google News app on Android and iOS."
 - name: search_appearance
   type: STRING
   mode: NULLABLE
-  description: How the search result appeared (e.g. normal result, translated result, video).
+  description: How the search result appeared (e.g. normal result, translated result,
+    video).
 - name: user_country_code
   type: STRING
   mode: NULLABLE
-  description: |-
-    Country from which the user was searching, in ISO-3166-1-alpha-3 format.
-    This will be null for anonymized Discover impressions.
+  description: 'Country from which the user was searching, in ISO-3166-1-alpha-3 format.
+
+    This will be null for anonymized Discover impressions.'
 - name: user_country
   type: STRING
   mode: NULLABLE
-  description: |-
-    Country from which the user was searching.
-    This will be null for anonymized Discover impressions.
+  description: 'Country from which the user was searching.
+
+    This will be null for anonymized Discover impressions.'
 - name: user_region
   type: STRING
   mode: NULLABLE
@@ -120,13 +126,15 @@
 - name: device_type
   type: STRING
   mode: NULLABLE
-  description: |-
-    The type of device on which the user was searching: Desktop, Mobile, or Tablet.
-    This will be null for Discover impressions.
+  description: 'The type of device on which the user was searching: Desktop, Mobile,
+    or Tablet.
+
+    This will be null for Discover impressions.'
 - name: impressions
   type: INTEGER
   mode: NULLABLE
-  description: The number of times that search results with a link to the page were shown to a user.
+  description: The number of times that search results with a link to the page were
+    shown to a user.
 - name: clicks
   type: INTEGER
   mode: NULLABLE
@@ -134,6 +142,7 @@
 - name: average_position
   type: FLOAT
   mode: NULLABLE
-  description: |-
-    The average position of the page in the search results, where `1` is the topmost position.
-    This will be null for Discover and Google News search impressions.
+  description: 'The average position of the page in the search results, where `1`
+    is the topmost position.
+
+    This will be null for Discover and Google News search impressions.'
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/google_search_console/search_impressions_by_site/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/google_search_console/search_impressions_by_site/schema.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/google_search_console/search_impressions_by_site/schema.yaml	2025-01-29 19:38:28.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/google_search_console/search_impressions_by_site/schema.yaml	2025-01-29 19:49:05.000000000 +0000
@@ -6,9 +6,10 @@
 - name: site_url
   type: STRING
   mode: NULLABLE
-  description: |-
-    For domain properties, this will be `sc-domain:` followed by the domain name.
-    For URL-prefix properties, it will be the full URL of the property definition.
+  description: 'For domain properties, this will be `sc-domain:` followed by the domain
+    name.
+
+    For URL-prefix properties, it will be the full URL of the property definition.'
 - name: site_domain_name
   type: STRING
   mode: NULLABLE
@@ -20,27 +21,23 @@
 - name: query_type
   type: STRING
   mode: NULLABLE
-  description: |-
-    Type of search query:
-      * Anonymized: Query was redacted by Google to protect the users' privacy.
-      * Brand: Query contained one or more Mozilla brand keywords.
-      * Non-Brand: Query didn't contain any Mozilla brand keywords.
-      * Unknown: Query couldn't be classified.
+  description: "Type of search query:\n  * Anonymized: Query was redacted by Google\
+    \ to protect the users' privacy.\n  * Brand: Query contained one or more Mozilla\
+    \ brand keywords.\n  * Non-Brand: Query didn't contain any Mozilla brand keywords.\n\
+    \  * Unknown: Query couldn't be classified."
 - name: is_anonymized
   type: BOOLEAN
   mode: NULLABLE
-  description: |-
-    Whether Google has anonymized the search impression to protect the users' privacy.
-    The `query` field will be null for anonymized search impressions.
+  description: 'Whether Google has anonymized the search impression to protect the
+    users'' privacy.
+
+    The `query` field will be null for anonymized search impressions.'
 - name: search_type
   type: STRING
   mode: NULLABLE
-  description: |-
-    Where the link was seen by the user:
-      * Web: In Google Search's default "All" tab.
-      * Image: In Google Search's "Images" tab.
-      * Video: In Google Search's "Videos" tab.
-      * News: In Google Search's "News" tab.
+  description: "Where the link was seen by the user:\n  * Web: In Google Search's\
+    \ default \"All\" tab.\n  * Image: In Google Search's \"Images\" tab.\n  * Video:\
+    \ In Google Search's \"Videos\" tab.\n  * News: In Google Search's \"News\" tab."
 - name: user_country_code
   type: STRING
   mode: NULLABLE
@@ -60,17 +57,20 @@
 - name: device_type
   type: STRING
   mode: NULLABLE
-  description: |-
-    The type of device on which the user was searching: Desktop, Mobile, or Tablet.
+  description: 'The type of device on which the user was searching: Desktop, Mobile,
+    or Tablet.'
 - name: impressions
   type: INTEGER
   mode: NULLABLE
-  description: The number of times that search results with at least one link to the site were shown to a user.
+  description: The number of times that search results with at least one link to the
+    site were shown to a user.
 - name: clicks
   type: INTEGER
   mode: NULLABLE
-  description: The number of times a user clicked at least one search result link to the site.
+  description: The number of times a user clicked at least one search result link
+    to the site.
 - name: average_top_position
   type: FLOAT
   mode: NULLABLE
-  description: The average top position of the site in the search results, where `1` is the topmost position.
+  description: The average top position of the site in the search results, where `1`
+    is the topmost position.
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_android/use_counters/schema.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/klar_android/use_counters/schema.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_android/use_counters/schema.yaml	2025-01-29 19:39:09.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/klar_android/use_counters/schema.yaml	2025-01-29 19:52:22.000000000 +0000
@@ -1,11 +1,11 @@
 fields:
 - name: normalized_app_id
-  type: STRING
   mode: NULLABLE
+  type: STRING
   description: App ID of the channel data was received from
 - name: normalized_channel
-  type: STRING
   mode: NULLABLE
+  type: STRING
   description: Normalized channel name
 - name: additional_properties
   type: STRING
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/engagement_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/engagement_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/engagement_v1/metadata.yaml	2025-01-29 19:41:10.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/engagement_v1/metadata.yaml	2025-01-29 19:55:50.000000000 +0000
@@ -1,13 +1,14 @@
 friendly_name: Profile / Client Engagement - Klar Android (Aggregated)
 description: |-
   Profile / Client Engagement (Klar Android) aggregated metrics
+
+  device_type and device_manufacturer fields were added on 2025-01-18.
 owners:
 - mozilla/kpi_table_reviewers
 - [email protected]
 labels:
   schedule: daily
   incremental: true
-  shredder_mitigation: true
   table_type: aggregate
   dag: bqetl_mobile_kpi_metrics
   owner1: kik
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/new_profiles_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/new_profiles_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/new_profiles_v1/metadata.yaml	2025-01-29 19:41:10.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/new_profiles_v1/metadata.yaml	2025-01-29 19:55:50.000000000 +0000
@@ -1,13 +1,14 @@
 friendly_name: New profiles - Klar Android (Aggregated)
 description: |-
   New profiles (Klar Android) aggregated metrics
+
+  device_type and device_manufacturer fields were added on 2025-01-18.
 owners:
 - mozilla/kpi_table_reviewers
 - [email protected]
 labels:
   schedule: daily
   incremental: true
-  shredder_mitigation: true
   table_type: aggregate
   dag: bqetl_mobile_kpi_metrics
   owner1: kik
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/retention_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/retention_v1/metadata.yaml
--- /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/retention_v1/metadata.yaml	2025-01-29 19:41:10.000000000 +0000
+++ /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/klar_android_derived/retention_v1/metadata.yaml	2025-01-29 19:55:50.000000000 +0000
@@ -6,13 +6,14 @@
   Client's attribute values as observed on the metric date.
 
   see: DENG-3183 for more information.
+
+  device_type and device_manufacturer fields were added on 2025-01-18.
 owners:
 - mozilla/kpi_table_reviewers
 - [email protected]
 labels:
   schedule: daily
   incremental: true
-  shredder_mitigation: true
   table_type: aggregate
   dag: bqetl_mobile_kpi_metrics
   owner1: kik
diff -bur --no-dereference --new-file /tmp/workspace/main-generated-sql/sql/moz-fx-data-shared-prod/klar_ios_derived/engagement_v1/metadata.yaml /tmp/workspace/generated-sql/sql/moz-fx-data-shared-prod/klar_ios_derived/engagement_v1/metadata.yaml
--- /tmp/workspace/main-generated-sq

⚠️ Only part of the diff is displayed.

Link to full diff

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Commit

There are no files selected for viewing

1 comment on commit `73cc651`

dataops-ci-bot commented on `73cc651` Jan 29, 2025

Commit

There are no files selected for viewing

1 comment on commit 73cc651

dataops-ci-bot commented on 73cc651 Jan 29, 2025

Choose a reason for hiding this comment

Integration report for "feat(DENG-6890): mobile kpi support metrics managed backfill 20250122 (#6847)"

sql.diff

1 comment on commit `73cc651`

dataops-ci-bot commented on `73cc651` Jan 29, 2025

`sql.diff`