From b01477e12473b14909d6f7b64ae5c4dfc5bdf1f9 Mon Sep 17 00:00:00 2001 From: kik-kik Date: Wed, 22 Jan 2025 15:41:33 +0100 Subject: [PATCH 1/4] feat: add backfill.yaml entries for fenix retention, new_profiles and engagement queries --- .../fenix_derived/engagement_v1/backfill.yaml | 7 +++++++ .../fenix_derived/new_profiles_v1/backfill.yaml | 7 +++++++ .../fenix_derived/retention_v1/backfill.yaml | 8 ++++++++ 3 files changed, 22 insertions(+) create mode 100644 sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1/backfill.yaml create mode 100644 sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1/backfill.yaml diff --git a/sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1/backfill.yaml b/sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1/backfill.yaml new file mode 100644 index 00000000000..87b7a17d4b2 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1/backfill.yaml @@ -0,0 +1,7 @@ +2025-01-22: + start_date: 2024-03-05 + end_date: 2025-01-21 + reason: Recompute partitions since DMA came into effect to include additional fields (DENG-6890). + watchers: + - kik@mozilla.com + status: Initiate diff --git a/sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1/backfill.yaml b/sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1/backfill.yaml new file mode 100644 index 00000000000..87b7a17d4b2 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1/backfill.yaml @@ -0,0 +1,7 @@ +2025-01-22: + start_date: 2024-03-05 + end_date: 2025-01-21 + reason: Recompute partitions since DMA came into effect to include additional fields (DENG-6890). + watchers: + - kik@mozilla.com + status: Initiate diff --git a/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/backfill.yaml b/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/backfill.yaml index f3a66e6765d..f1c68e5ce62 100644 --- a/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/backfill.yaml +++ b/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/backfill.yaml @@ -1,3 +1,11 @@ +2025-01-22: + start_date: 2024-03-05 + end_date: 2025-01-21 + reason: Recompute partitions since DMA came into effect to include additional fields (DENG-6890). + watchers: + - kik@mozilla.com + status: Initiate + 2024-05-31: start_date: 2021-01-01 end_date: 2024-05-31 From b97b0f094c03f17256c4092bfed33af28da2ff47 Mon Sep 17 00:00:00 2001 From: kik-kik Date: Wed, 22 Jan 2025 15:44:28 +0100 Subject: [PATCH 2/4] feat: add backfill.yaml entries for firefox_ios retention, new_profiles and engagement queries --- .../firefox_ios_derived/engagement_v1/backfill.yaml | 7 +++++++ .../firefox_ios_derived/new_profiles_v1/backfill.yaml | 7 +++++++ .../firefox_ios_derived/retention_v1/backfill.yaml | 8 ++++++++ 3 files changed, 22 insertions(+) create mode 100644 sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1/backfill.yaml create mode 100644 sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1/backfill.yaml diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1/backfill.yaml b/sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1/backfill.yaml new file mode 100644 index 00000000000..87b7a17d4b2 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1/backfill.yaml @@ -0,0 +1,7 @@ +2025-01-22: + start_date: 2024-03-05 + end_date: 2025-01-21 + reason: Recompute partitions since DMA came into effect to include additional fields (DENG-6890). + watchers: + - kik@mozilla.com + status: Initiate diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1/backfill.yaml b/sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1/backfill.yaml new file mode 100644 index 00000000000..87b7a17d4b2 --- /dev/null +++ b/sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1/backfill.yaml @@ -0,0 +1,7 @@ +2025-01-22: + start_date: 2024-03-05 + end_date: 2025-01-21 + reason: Recompute partitions since DMA came into effect to include additional fields (DENG-6890). + watchers: + - kik@mozilla.com + status: Initiate diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/retention_v1/backfill.yaml b/sql/moz-fx-data-shared-prod/firefox_ios_derived/retention_v1/backfill.yaml index fbcd7919f34..0acb79a384c 100644 --- a/sql/moz-fx-data-shared-prod/firefox_ios_derived/retention_v1/backfill.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_ios_derived/retention_v1/backfill.yaml @@ -1,3 +1,11 @@ +2025-01-22: + start_date: 2024-03-05 + end_date: 2025-01-21 + reason: Recompute partitions since DMA came into effect to include additional fields (DENG-6890). + watchers: + - kik@mozilla.com + status: Initiate + 2024-06-04: start_date: 2021-01-01 end_date: 2024-06-04 From 4e11d89ba6339c004aaf865b8ef36cef1d2c4b8f Mon Sep 17 00:00:00 2001 From: kik-kik Date: Wed, 22 Jan 2025 15:53:37 +0100 Subject: [PATCH 3/4] feat: add a note to metadata that the new fields are available on firefox ios and fenix starting 2024-03-04 --- .../templates/engagement.metadata.yaml | 6 ++++++ .../templates/new_profiles.metadata.yaml | 6 ++++++ .../templates/retention.metadata.yaml | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/sql_generators/mobile_kpi_support_metrics/templates/engagement.metadata.yaml b/sql_generators/mobile_kpi_support_metrics/templates/engagement.metadata.yaml index d13bf8f5dbb..983e1d59ae9 100644 --- a/sql_generators/mobile_kpi_support_metrics/templates/engagement.metadata.yaml +++ b/sql_generators/mobile_kpi_support_metrics/templates/engagement.metadata.yaml @@ -2,6 +2,12 @@ friendly_name: Profile / Client Engagement - {{ friendly_name }} (Aggregated) description: |- Profile / Client Engagement ({{ friendly_name }}) aggregated metrics + {% if app_name in ['fenix', 'firefox_ios'] -%} + device_type and device_manufacturer fields were added in 2024-03-05. + {%- else -%} + device_type and device_manufacturer fields were added on 2025-01-18. + {%- endif %} + owners: - mozilla/kpi_table_reviewers - kik@mozilla.com diff --git a/sql_generators/mobile_kpi_support_metrics/templates/new_profiles.metadata.yaml b/sql_generators/mobile_kpi_support_metrics/templates/new_profiles.metadata.yaml index b577d3f5e39..d1312516c31 100644 --- a/sql_generators/mobile_kpi_support_metrics/templates/new_profiles.metadata.yaml +++ b/sql_generators/mobile_kpi_support_metrics/templates/new_profiles.metadata.yaml @@ -2,6 +2,12 @@ friendly_name: New profiles - {{ friendly_name }} (Aggregated) description: |- New profiles ({{ friendly_name }}) aggregated metrics + {% if app_name in ['fenix', 'firefox_ios'] -%} + device_type and device_manufacturer fields were added in 2024-03-05. + {%- else -%} + device_type and device_manufacturer fields were added on 2025-01-18. + {%- endif %} + owners: - mozilla/kpi_table_reviewers - kik@mozilla.com diff --git a/sql_generators/mobile_kpi_support_metrics/templates/retention.metadata.yaml b/sql_generators/mobile_kpi_support_metrics/templates/retention.metadata.yaml index df4128c8c78..79598e210b5 100644 --- a/sql_generators/mobile_kpi_support_metrics/templates/retention.metadata.yaml +++ b/sql_generators/mobile_kpi_support_metrics/templates/retention.metadata.yaml @@ -7,6 +7,12 @@ description: |- see: DENG-3183 for more information. + {% if app_name in ['fenix', 'firefox_ios'] -%} + device_type and device_manufacturer fields were added in 2024-03-05. + {%- else -%} + device_type and device_manufacturer fields were added on 2025-01-18. + {%- endif %} + owners: - mozilla/kpi_table_reviewers - kik@mozilla.com From dcc66ea1ea655f78ae57b389a0e7fc3061117007 Mon Sep 17 00:00:00 2001 From: kik-kik Date: Thu, 23 Jan 2025 11:35:59 +0100 Subject: [PATCH 4/4] feat: add shredder_mitigation flag to the new backfills entries, but set it to false also in metadata for now. --- .../fenix_derived/engagement_v1/backfill.yaml | 1 + .../fenix_derived/new_profiles_v1/backfill.yaml | 1 + .../fenix_derived/retention_v1/backfill.yaml | 1 + .../firefox_ios_derived/engagement_v1/backfill.yaml | 1 + .../firefox_ios_derived/new_profiles_v1/backfill.yaml | 1 + .../firefox_ios_derived/retention_v1/backfill.yaml | 1 + .../templates/engagement.metadata.yaml | 2 +- .../templates/new_profiles.metadata.yaml | 2 +- .../templates/retention.metadata.yaml | 2 +- 9 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1/backfill.yaml b/sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1/backfill.yaml index 87b7a17d4b2..c97b36fbbb4 100644 --- a/sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1/backfill.yaml +++ b/sql/moz-fx-data-shared-prod/fenix_derived/engagement_v1/backfill.yaml @@ -5,3 +5,4 @@ watchers: - kik@mozilla.com status: Initiate + shredder_mitigation: false diff --git a/sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1/backfill.yaml b/sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1/backfill.yaml index 87b7a17d4b2..c97b36fbbb4 100644 --- a/sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1/backfill.yaml +++ b/sql/moz-fx-data-shared-prod/fenix_derived/new_profiles_v1/backfill.yaml @@ -5,3 +5,4 @@ watchers: - kik@mozilla.com status: Initiate + shredder_mitigation: false diff --git a/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/backfill.yaml b/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/backfill.yaml index f1c68e5ce62..92e587dffe9 100644 --- a/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/backfill.yaml +++ b/sql/moz-fx-data-shared-prod/fenix_derived/retention_v1/backfill.yaml @@ -5,6 +5,7 @@ watchers: - kik@mozilla.com status: Initiate + shredder_mitigation: false 2024-05-31: start_date: 2021-01-01 diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1/backfill.yaml b/sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1/backfill.yaml index 87b7a17d4b2..c97b36fbbb4 100644 --- a/sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1/backfill.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_ios_derived/engagement_v1/backfill.yaml @@ -5,3 +5,4 @@ watchers: - kik@mozilla.com status: Initiate + shredder_mitigation: false diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1/backfill.yaml b/sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1/backfill.yaml index 87b7a17d4b2..c97b36fbbb4 100644 --- a/sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1/backfill.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_ios_derived/new_profiles_v1/backfill.yaml @@ -5,3 +5,4 @@ watchers: - kik@mozilla.com status: Initiate + shredder_mitigation: false diff --git a/sql/moz-fx-data-shared-prod/firefox_ios_derived/retention_v1/backfill.yaml b/sql/moz-fx-data-shared-prod/firefox_ios_derived/retention_v1/backfill.yaml index 0acb79a384c..e3a960e8bf7 100644 --- a/sql/moz-fx-data-shared-prod/firefox_ios_derived/retention_v1/backfill.yaml +++ b/sql/moz-fx-data-shared-prod/firefox_ios_derived/retention_v1/backfill.yaml @@ -5,6 +5,7 @@ watchers: - kik@mozilla.com status: Initiate + shredder_mitigation: false 2024-06-04: start_date: 2021-01-01 diff --git a/sql_generators/mobile_kpi_support_metrics/templates/engagement.metadata.yaml b/sql_generators/mobile_kpi_support_metrics/templates/engagement.metadata.yaml index 983e1d59ae9..1e6561d2fa4 100644 --- a/sql_generators/mobile_kpi_support_metrics/templates/engagement.metadata.yaml +++ b/sql_generators/mobile_kpi_support_metrics/templates/engagement.metadata.yaml @@ -14,7 +14,7 @@ owners: labels: schedule: daily incremental: true - shredder_mitigation: true + shredder_mitigation: false table_type: aggregate scheduling: dag_name: bqetl_mobile_kpi_metrics diff --git a/sql_generators/mobile_kpi_support_metrics/templates/new_profiles.metadata.yaml b/sql_generators/mobile_kpi_support_metrics/templates/new_profiles.metadata.yaml index d1312516c31..cd5658b53c0 100644 --- a/sql_generators/mobile_kpi_support_metrics/templates/new_profiles.metadata.yaml +++ b/sql_generators/mobile_kpi_support_metrics/templates/new_profiles.metadata.yaml @@ -14,7 +14,7 @@ owners: labels: schedule: daily incremental: true - shredder_mitigation: true + shredder_mitigation: false table_type: aggregate scheduling: dag_name: bqetl_mobile_kpi_metrics diff --git a/sql_generators/mobile_kpi_support_metrics/templates/retention.metadata.yaml b/sql_generators/mobile_kpi_support_metrics/templates/retention.metadata.yaml index 79598e210b5..372ca4ce76f 100644 --- a/sql_generators/mobile_kpi_support_metrics/templates/retention.metadata.yaml +++ b/sql_generators/mobile_kpi_support_metrics/templates/retention.metadata.yaml @@ -19,7 +19,7 @@ owners: labels: schedule: daily incremental: true - shredder_mitigation: true + shredder_mitigation: false table_type: aggregate scheduling: dag_name: bqetl_mobile_kpi_metrics