Create aggregate tables for Firefox Health Indicator dashboard (#6665)

* Create 2 new tables from Firefox health indicator dashboard

* Fix dag name

* Add schema.yaml

* Fix dags.yaml indentation

* Add cluster column

* Add new table fx_health_ind_win_instll_by_instll_typ_v1

* Add fx_health_ind_win_uninstll_v1

* Update metadata.yaml

* Fix query

* Update labels

* Add fx_health_ind_mau_per_os

* Make partition filter not required

* Update start date

dags.yaml

@@ -1959,3 +1959,22 @@ bqetl_newtab_engagement_hourly:
     retry_delay: 5m
   tags:
     - impact/tier_2
+
+bqetl_fx_health_ind_dashboard:
+  description: |
+    This DAG builds aggregate tables used in the Firefox Health dashboard
+  default_args:
+    depends_on_past: false
+    owner: [email protected]
+    email:
+      - [email protected]
+      - [email protected]
+    email_on_failure: true
+    email_on_retry: false
+    start_date: "2024-12-13"
+    retries: 2
+    retry_delay: 5m
+  tags:
+    - impact/tier_2
+  repo: bigquery-etl
+  schedule_interval: 0 16 * * *

sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_desktop_dau_by_device_type/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.telemetry.fx_health_ind_desktop_dau_by_device_type`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.telemetry_derived.fx_health_ind_desktop_dau_by_device_type_v1`

sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_fqueze_cpu_info/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.telemetry.fx_health_ind_fqueze_cpu_info`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.telemetry_derived.fx_health_ind_fqueze_cpu_info_v1`

sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_mau_per_os/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.telemetry.fx_health_ind_mau_per_os`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.telemetry_derived.fx_health_ind_mau_per_os_v1`

sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_win_instll_by_instll_typ/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.telemetry.fx_health_ind_win_instll_by_instll_typ`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.telemetry_derived.fx_health_ind_win_instll_by_instll_typ_v1`

sql/moz-fx-data-shared-prod/telemetry/fx_health_ind_win_uninstll/view.sql

@@ -0,0 +1,7 @@
+CREATE OR REPLACE VIEW
+  `moz-fx-data-shared-prod.telemetry.fx_health_ind_win_uninstll`
+AS
+SELECT
+  *
+FROM
+  `moz-fx-data-shared-prod.telemetry_derived.fx_health_ind_win_uninstll_v1`

sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_desktop_dau_by_device_type_v1/metadata.yaml

@@ -0,0 +1,23 @@
+friendly_name: Fx Health Ind Desktop Dau By Device Type
+description: |-
+  Aggregate table that calculates desktop DAU by device type
+owners:
+- [email protected]
+labels:
+  incremental: true
+  owner1: [email protected]
+  table_type: aggregate
+  shredder_mitigation: true
+scheduling:
+  dag_name: bqetl_fx_health_ind_dashboard
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+    expiration_days: null
+  range_partitioning: null
+  clustering:
+    fields:
+    - TDP
+references: {}

sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_desktop_dau_by_device_type_v1/query.sql

@@ -0,0 +1,15 @@
+SELECT
+  DATE(t.submission_timestamp) AS submission_date,
+  cpu.cpu_type AS TDP,
+  100 * COUNT(DISTINCT(client_id)) AS users
+FROM
+  `moz-fx-data-shared-prod.telemetry.main_1pct` AS t
+JOIN
+  `moz-fx-data-shared-prod.telemetry_derived.fx_health_ind_fqueze_cpu_info_v1` AS cpu
+  ON cpu.cpu_name = environment.system.cpu.name
+  AND environment.system.cpu.name IS NOT NULL
+WHERE
+  DATE(t.submission_timestamp) = @submission_date
+GROUP BY
+  DATE(t.submission_timestamp),
+  cpu.cpu_type

sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_desktop_dau_by_device_type_v1/schema.yaml

@@ -0,0 +1,13 @@
+fields:
+- mode: NULLABLE
+  name: submission_date
+  type: DATE
+  description: Submission Date
+- mode: NULLABLE
+  name: TDP
+  type: STRING
+  description: TDP
+- mode: NULLABLE
+  name: users
+  type: INTEGER
+  description: Number of Users

sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_fqueze_cpu_info_v1/metadata.yaml

@@ -0,0 +1,14 @@
+friendly_name: Fx Health Ind Fqueze Cpu Info
+description: |-
+  Static table of CPU Information used in Firefox Health Indicators dashboard
+owners:
+- [email protected]
+labels:
+  incremental: true
+  owner1: [email protected]
+bigquery:
+  range_partitioning: null
+  clustering:
+    fields:
+    - cpu_name
+references: {}

sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_fqueze_cpu_info_v1/schema.yaml

@@ -0,0 +1,17 @@
+fields:
+- mode: REQUIRED
+  name: cpu_name
+  type: STRING
+  description: CPU Name
+- mode: REQUIRED
+  name: cpu_tdp
+  type: FLOAT
+  description: CPU TDP
+- mode: REQUIRED
+  name: cpu_cores
+  type: INTEGER
+  description: CPU Cores
+- mode: NULLABLE
+  name: cpu_type
+  type: STRING
+  description: CPU Type

sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_os_v1/metadata.yaml

@@ -0,0 +1,22 @@
+friendly_name: Fx Health Indicator Mau by OS
+description: |-
+  Aggregate table calculating MAU and DAU per OS
+owners:
+- [email protected]
+labels:
+  incremental: true
+  owner1: [email protected]
+  table_type: aggregate
+scheduling:
+  dag_name: bqetl_fx_health_ind_dashboard
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+    expiration_days: null
+  range_partitioning: null
+  clustering:
+    fields:
+    - os
+references: {}

sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_os_v1/query.sql

@@ -0,0 +1,54 @@
+WITH sample_cte AS (
+  SELECT
+    submission_date,
+    os,
+    SUM(dau) AS tot_dau,
+    SUM(mau) AS tot_mau
+  FROM
+    `moz-fx-data-shared-prod.telemetry.active_users_aggregates` --telemetry.firefox_desktop_exact_mau28_by_client_count_dimensions
+  WHERE
+    app_name = 'Firefox Desktop'
+    AND submission_date
+    BETWEEN DATE_SUB(@submission_date, INTERVAL 6 DAY)
+    AND @submission_date
+  GROUP BY
+    submission_date,
+    os
+  HAVING
+    SUM(mau) > 1000
+),
+smoothed AS (
+  SELECT
+    *,
+    AVG(tot_dau) OVER (
+      PARTITION BY
+        os
+      ORDER BY
+        submission_date
+      ROWS BETWEEN
+        6 PRECEDING
+        AND 0 FOLLOWING
+    ) AS smoothed_dau,
+    COUNT(1) OVER (
+      PARTITION BY
+        os
+      ORDER BY
+        submission_date
+      ROWS BETWEEN
+        6 PRECEDING
+        AND 0 FOLLOWING
+    ) AS nbr_Days_included
+  FROM
+    sample_cte
+)
+SELECT
+  submission_date,
+  os,
+  tot_dau AS dau,
+  tot_mau AS mau,
+  smoothed_dau,
+  smoothed_dau / tot_mau AS ER
+FROM
+  smoothed
+WHERE
+  nbr_days_included = 7 --only include those operating systems that have at least 1000 MAU on all 7 days

sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_mau_per_os_v1/schema.yaml

@@ -0,0 +1,25 @@
+fields:
+- mode: NULLABLE
+  name: submission_date
+  type: DATE
+  description: Submission Date
+- mode: NULLABLE
+  name: os
+  type: STRING
+  description: Operating System
+- mode: NULLABLE
+  name: dau
+  type: INTEGER
+  description: DAU
+- mode: NULLABLE
+  name: mau
+  type: INTEGER
+  description: MAU
+- mode: NULLABLE
+  name: smoothed_dau
+  type: FLOAT
+  description: Smoothed DAU
+- mode: NULLABLE
+  name: ER
+  type: FLOAT
+  description: ER - Smoothed DAU Divided by MAU

sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_win_instll_by_instll_typ_v1/metadata.yaml

@@ -0,0 +1,23 @@
+friendly_name: Firefox Health Indicator Windows Installs
+description: |-
+  Aggregate table of windows installs by installer type, used in Firefox Health dashboard
+owners:
+- [email protected]
+labels:
+  incremental: true
+  owner1: [email protected]
+  table_type: aggregate
+  shredder_mitigation: true
+scheduling:
+  dag_name: bqetl_fx_health_ind_dashboard
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+    expiration_days: null
+  range_partitioning: null
+  clustering:
+    fields:
+    - installer_type
+references: {}

sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_win_instll_by_instll_typ_v1/query.sql

@@ -0,0 +1,12 @@
+SELECT
+  CAST(submission_timestamp AS DATE) AS submission_date,
+  installer_type,
+  COUNT(1) AS install_count,
+FROM
+  `moz-fx-data-shared-prod.firefox_installer.install`
+WHERE
+  DATE(submission_timestamp) = @submission_date
+  AND update_channel = 'release'
+GROUP BY
+  CAST(submission_timestamp AS DATE),
+  installer_type

sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_win_instll_by_instll_typ_v1/schema.yaml

@@ -0,0 +1,13 @@
+fields:
+- mode: NULLABLE
+  name: submission_date
+  type: DATE
+  description: Submission Date
+- mode: NULLABLE
+  name: installer_type
+  type: STRING
+  description: Installer Type
+- mode: NULLABLE
+  name: install_count
+  type: INTEGER
+  description: Install Count

sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_win_uninstll_v1/metadata.yaml

@@ -0,0 +1,19 @@
+friendly_name: Fx Health Ind Win Uninstll
+description: |-
+  Aggregate view of Windows uninstalls by day
+owners:
+- [email protected]
+labels:
+  incremental: true
+  owner1: [email protected]
+  table_type: aggregate
+scheduling:
+  dag_name: bqetl_fx_health_ind_dashboard
+bigquery:
+  time_partitioning:
+    type: day
+    field: submission_date
+    require_partition_filter: false
+    expiration_days: null
+  range_partitioning: null
+references: {}

sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_win_uninstll_v1/query.sql

@@ -0,0 +1,11 @@
+SELECT
+  DATE(submission_timestamp) AS submission_date,
+  COUNT(DISTINCT(client_id)) AS uninstall_client_count
+FROM
+  `moz-fx-data-shared-prod.telemetry.uninstall`
+WHERE
+  DATE(submission_timestamp) = @submission_date
+  AND application.name = 'Firefox'
+  AND application.channel = 'release'
+GROUP BY
+  DATE(submission_timestamp)

sql/moz-fx-data-shared-prod/telemetry_derived/fx_health_ind_win_uninstll_v1/schema.yaml

@@ -0,0 +1,9 @@
+fields:
+- mode: NULLABLE
+  name: submission_date
+  type: DATE
+  description: Submission Date
+- mode: NULLABLE
+  name: uninstall_client_count
+  type: INTEGER
+  description: Count of Clients with Uninstalls