Merge pull request #1493 from GSA/2199-add-pending-message-data-to-da…

…ily-and-user_daily-stats

add pending stats to daily stats endpoint

app/dao/services_dao.py

@@ -469,6 +469,35 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date):
     start_date = get_midnight_in_utc(start_date)
     end_date = get_midnight_in_utc(end_date + timedelta(days=1))
 
+    total_substmt = (
+        select(
+            func.date_trunc("day", NotificationAllTimeView.created_at).label("day"),
+            Job.notification_count.label("notification_count"),
+        )
+        .join(Job, NotificationAllTimeView.job_id == Job.id)
+        .where(
+            NotificationAllTimeView.service_id == service_id,
+            NotificationAllTimeView.key_type != KeyType.TEST,
+            NotificationAllTimeView.created_at >= start_date,
+            NotificationAllTimeView.created_at < end_date,
+        )
+        .group_by(
+            Job.id,
+            Job.notification_count,
+            func.date_trunc("day", NotificationAllTimeView.created_at),
+        )
+        .subquery()
+    )
+
+    total_stmt = select(
+        total_substmt.c.day,
+        func.sum(total_substmt.c.notification_count).label("total_notifications"),
+    ).group_by(total_substmt.c.day)
+
+    total_notifications = {
+        row.day: row.total_notifications for row in db.session.execute(total_stmt).all()
+    }
+
     stmt = (
         select(
             NotificationAllTimeView.notification_type,
@@ -488,7 +517,10 @@ def dao_fetch_stats_for_service_from_days(service_id, start_date, end_date):
             func.date_trunc("day", NotificationAllTimeView.created_at),
         )
     )
-    return db.session.execute(stmt).all()
+
+    data = db.session.execute(stmt).all()
+
+    return total_notifications, data
 
 
 def dao_fetch_stats_for_service_from_days_for_user(
@@ -497,14 +529,43 @@ def dao_fetch_stats_for_service_from_days_for_user(
     start_date = get_midnight_in_utc(start_date)
     end_date = get_midnight_in_utc(end_date + timedelta(days=1))
 
+    total_substmt = (
+        select(
+            func.date_trunc("day", NotificationAllTimeView.created_at).label("day"),
+            Job.notification_count.label("notification_count"),
+        )
+        .join(Job, NotificationAllTimeView.job_id == Job.id)
+        .where(
+            NotificationAllTimeView.service_id == service_id,
+            NotificationAllTimeView.key_type != KeyType.TEST,
+            NotificationAllTimeView.created_at >= start_date,
+            NotificationAllTimeView.created_at < end_date,
+            NotificationAllTimeView.created_by_id == user_id,
+        )
+        .group_by(
+            Job.id,
+            Job.notification_count,
+            func.date_trunc("day", NotificationAllTimeView.created_at),
+        )
+        .subquery()
+    )
+
+    total_stmt = select(
+        total_substmt.c.day,
+        func.sum(total_substmt.c.notification_count).label("total_notifications"),
+    ).group_by(total_substmt.c.day)
+
+    total_notifications = {
+        row.day: row.total_notifications for row in db.session.execute(total_stmt).all()
+    }
+
     stmt = (
         select(
             NotificationAllTimeView.notification_type,
             NotificationAllTimeView.status,
             func.date_trunc("day", NotificationAllTimeView.created_at).label("day"),
             func.count(NotificationAllTimeView.id).label("count"),
         )
-        .select_from(NotificationAllTimeView)
         .where(
             NotificationAllTimeView.service_id == service_id,
             NotificationAllTimeView.key_type != KeyType.TEST,
@@ -518,7 +579,10 @@ def dao_fetch_stats_for_service_from_days_for_user(
             func.date_trunc("day", NotificationAllTimeView.created_at),
         )
     )
-    return db.session.execute(stmt).scalars().all()
+
+    data = db.session.execute(stmt).all()
+
+    return total_notifications, data
 
 
 def dao_fetch_todays_stats_for_all_services(
@@ -734,7 +798,9 @@ def fetch_notification_stats_for_service_by_month_by_user(
     return db.session.execute(stmt).all()
 
 
-def get_specific_days_stats(data, start_date, days=None, end_date=None):
+def get_specific_days_stats(
+    data, start_date, days=None, end_date=None, total_notifications=None
+):
     if days is not None and end_date is not None:
         raise ValueError("Only set days OR set end_date, not both.")
     elif days is not None:
@@ -745,13 +811,19 @@ def get_specific_days_stats(data, start_date, days=None, end_date=None):
         raise ValueError("Either days or end_date must be set.")
 
     grouped_data = {date: [] for date in gen_range} | {
-        day: [row for row in data if row.day.date() == day]
-        for day in {item.day.date() for item in data}
+        day: [row for row in data if row.day == day]
+        for day in {item.day for item in data}
     }
 
     stats = {
-        day.strftime("%Y-%m-%d"): statistics.format_statistics(rows)
+        day.strftime("%Y-%m-%d"): statistics.format_statistics(
+            rows,
+            total_notifications=(
+                total_notifications.get(day, 0)
+                if total_notifications is not None
+                else None
+            ),
+        )
         for day, rows in grouped_data.items()
     }
-
     return stats

app/enums.py

@@ -211,3 +211,4 @@ class StatisticsType(StrEnum):
     REQUESTED = "requested"
     DELIVERED = "delivered"
     FAILURE = "failure"
+    PENDING = "pending"

app/service/rest.py

@@ -232,9 +232,18 @@ def get_service_statistics_for_specific_days(service_id, start, days=1):
     end_date = datetime.strptime(start, "%Y-%m-%d")
     start_date = end_date - timedelta(days=days - 1)
 
-    results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date)
+    total_notifications, results = dao_fetch_stats_for_service_from_days(
+        service_id,
+        start_date,
+        end_date,
+    )
 
-    stats = get_specific_days_stats(results, start_date, days=days)
+    stats = get_specific_days_stats(
+        results,
+        start_date,
+        days=days,
+        total_notifications=total_notifications,
+    )
 
     return stats
 
@@ -261,12 +270,16 @@ def get_service_statistics_for_specific_days_by_user(
     end_date = datetime.strptime(start, "%Y-%m-%d")
     start_date = end_date - timedelta(days=days - 1)
 
-    results = dao_fetch_stats_for_service_from_days_for_user(
+    total_notifications, results = dao_fetch_stats_for_service_from_days_for_user(
         service_id, start_date, end_date, user_id
     )
 
-    stats = get_specific_days_stats(results, start_date, days=days)
-
+    stats = get_specific_days_stats(
+        results,
+        start_date,
+        days=days,
+        total_notifications=total_notifications,
+    )
     return stats
 
 
@@ -676,11 +689,11 @@ def get_single_month_notification_stats_by_user(service_id, user_id):
     month_year = datetime(year, month, 10, 00, 00, 00)
     start_date, end_date = get_month_start_and_end_date_in_utc(month_year)
 
-    results = dao_fetch_stats_for_service_from_days_for_user(
+    total_notifications, results = dao_fetch_stats_for_service_from_days_for_user(
         service_id, start_date, end_date, user_id
     )
 
-    stats = get_specific_days_stats(results, start_date, end_date=end_date)
+    stats = get_specific_days_stats(results, start_date, end_date=end_date, total_notifications=total_notifications,)
     return jsonify(stats)
 
 
@@ -700,7 +713,9 @@ def get_single_month_notification_stats_for_service(service_id):
     month_year = datetime(year, month, 10, 00, 00, 00)
     start_date, end_date = get_month_start_and_end_date_in_utc(month_year)
 
-    results = dao_fetch_stats_for_service_from_days(service_id, start_date, end_date)
+    __, results = dao_fetch_stats_for_service_from_days(
+        service_id, start_date, end_date
+    )
 
     stats = get_specific_days_stats(results, start_date, end_date=end_date)
     return jsonify(stats)

app/service/statistics.py

@@ -2,10 +2,16 @@
 from datetime import datetime
 
 from app.dao.date_util import get_months_for_financial_year
-from app.enums import KeyType, NotificationStatus, StatisticsType, TemplateType
+from app.enums import (
+    KeyType,
+    NotificationStatus,
+    NotificationType,
+    StatisticsType,
+    TemplateType,
+)
 
 
-def format_statistics(statistics):
+def format_statistics(statistics, total_notifications=None):
     # statistics come in a named tuple with uniqueness from 'notification_type', 'status' - however missing
     # statuses/notification types won't be represented and the status types need to be simplified/summed up
     # so we can return emails/sms * created, sent, and failed
@@ -14,11 +20,27 @@ def format_statistics(statistics):
         # any row could be null, if the service either has no notifications in the notifications table,
         # or no historical data in the ft_notification_status table.
         if row.notification_type:
-            _update_statuses_from_row(counts[row.notification_type], row)
+            _update_statuses_from_row(
+                counts[row.notification_type],
+                row,
+            )
+
+        if NotificationType.SMS in counts and total_notifications is not None:
+            sms_dict = counts[NotificationType.SMS]
+            delivered_count = sms_dict[StatisticsType.DELIVERED]
+            failed_count = sms_dict[StatisticsType.FAILURE]
+            sms_dict[StatisticsType.PENDING] = calculate_pending_stats(
+                delivered_count, failed_count, total_notifications
+            )
 
     return counts
 
 
+def calculate_pending_stats(delivered_count, failed_count, total_notifications):
+    pending_count = total_notifications - (delivered_count + failed_count)
+    return max(0, pending_count)
+
+
 def format_admin_stats(statistics):
     counts = create_stats_dict()