Merge pull request #1440 from GSA/main

04/17/2024 Production Deploy

.github/workflows/checks.yml

@@ -157,9 +157,11 @@ jobs:
       - uses: ./.github/actions/setup-project
       - name: Create requirements.txt
         run: poetry export --without-hashes --format=requirements.txt > requirements.txt
-      - uses: pypa/[email protected].6
+      - uses: pypa/[email protected].8
         with:
           inputs: requirements.txt
+          ignore-vulns: |
+            GHSA-w3h3-4rj7-4ph4
       - name: Run npm audit
         run: make npm-audit
 

Makefile

@@ -82,7 +82,7 @@ dead-code:
 .PHONY: e2e-test
 e2e-test: export NEW_RELIC_ENVIRONMENT=test
 e2e-test: ## Run end-to-end integration tests; note that --browser webkit isn't currently working
-	poetry run pytest -v --browser chromium --browser firefox tests/end_to_end
+	poetry run pytest -vv --browser chromium  --browser firefox tests/end_to_end
 
 .PHONY: js-lint
 js-lint: ## Run javascript linting scanners

app/__init__.py

@@ -52,6 +52,7 @@
     format_datetime_human,
     format_datetime_normal,
     format_datetime_relative,
+    format_datetime_scheduled_notification,
     format_datetime_table,
     format_day_of_week,
     format_delta,
@@ -551,6 +552,7 @@ def add_template_filters(application):
         format_datetime,
         format_datetime_24h,
         format_datetime_normal,
+        format_datetime_scheduled_notification,
         format_datetime_table,
         valid_phone_number,
         linkable_name,

app/formatters.py

@@ -22,7 +22,7 @@
 from notifications_utils.take import Take
 
 from app.utils.csv import get_user_preferred_timezone
-from app.utils.time import parse_dt, parse_naive_dt
+from app.utils.time import parse_naive_dt
 
 
 def apply_html_class(tags, html_file):
@@ -93,16 +93,34 @@ def format_datetime_normal(date):
     )
 
 
+def format_datetime_scheduled_notification(date):
+    # e.g. April 09, 2024 at 04:00 PM US/Eastern.
+    # Everything except scheduled notifications, the time is always "now".
+    # Scheduled notifications are the exception to the rule.
+    # Here we are formating and displaying the datetime without converting datetime to a different timezone.
+
+    datetime_obj = parse_naive_dt(date)
+
+    format_time_without_tz = datetime_obj.replace(tzinfo=timezone.utc).strftime(
+        "%I:%M %p"
+    )
+    return "{} at {} {}".format(
+        format_date_normal(date), format_time_without_tz, get_user_preferred_timezone()
+    )
+
+
 def format_datetime_table(date):
     # example:  03-18-2024 at 04:53 PM, intended for datetimes in tables
     return "{} at {}".format(format_date_numeric(date), format_time_12h(date))
 
 
 def format_time_12h(date):
-    date = parse_dt(date)
+    date = parse_naive_dt(date)
 
     preferred_tz = pytz.timezone(get_user_preferred_timezone())
-    return date.astimezone(preferred_tz).strftime("%I:%M %p")
+    return (
+        date.replace(tzinfo=timezone.utc).astimezone(preferred_tz).strftime("%I:%M %p")
+    )
 
 
 def format_datetime_relative(date):

app/main/views/dashboard.py

@@ -134,11 +134,17 @@ def get_monthly_template_stats(month_name, stats):
         months=months,
         stats=stats,
         most_used_template_count=max(
-            max(
-                (template["requested_count"] for template in month["templates_used"]),
-                default=0,
-            )
-            for month in months
+            (
+                max(
+                    (
+                        template["requested_count"]
+                        for template in month["templates_used"]
+                    ),
+                    default=0,
+                )
+                for month in months
+            ),
+            default=0,
         ),
         years=get_tuples_of_financial_years(
             partial(url_for, ".template_usage", service_id=service_id),
@@ -155,31 +161,16 @@ def usage(service_id):
     year, current_financial_year = requested_and_current_financial_year(request)
 
     free_sms_allowance = billing_api_client.get_free_sms_fragment_limit_for_year(
-        service_id, year
+        service_id
     )
+
     units = billing_api_client.get_monthly_usage_for_service(service_id, year)
+
     yearly_usage = billing_api_client.get_annual_usage_for_service(service_id, year)
 
     more_stats = format_monthly_stats_to_list(
         service_api_client.get_monthly_notification_stats(service_id, year)["data"]
     )
-    if year == current_financial_year:
-        # This includes Oct, Nov, Dec
-        # but we don't need next year's data yet
-        more_stats = [
-            month
-            for month in more_stats
-            if month["name"] in ["October", "November", "December"]
-        ]
-    elif year == (current_financial_year + 1):
-        # This is all the other months
-        # and we need last year's data
-        more_stats = [
-            month
-            for month in more_stats
-            if month["name"] not in ["October", "November", "December"]
-        ]
-
     return render_template(
         "views/usage.html",
         months=list(get_monthly_usage_breakdown(year, units, more_stats)),
@@ -341,8 +332,15 @@ def get_dashboard_partials(service_id):
     dashboard_totals = (get_dashboard_totals(stats),)
     free_sms_allowance = billing_api_client.get_free_sms_fragment_limit_for_year(
         current_service.id,
-        get_current_financial_year(),
     )
+    # These 2 calls will update the dashboard sms allowance count while in trial mode.
+    billing_api_client.get_monthly_usage_for_service(
+        service_id, get_current_financial_year()
+    )
+    billing_api_client.create_or_update_free_sms_fragment_limit(
+        service_id, free_sms_fragment_limit=free_sms_allowance
+    )
+
     yearly_usage = billing_api_client.get_annual_usage_for_service(
         service_id,
         get_current_financial_year(),
@@ -433,13 +431,7 @@ def aggregate_status_types(counts_dict):
 
 
 def get_months_for_financial_year(year, time_format="%B"):
-    return [
-        month.strftime(time_format)
-        for month in (
-            get_months_for_year(10, 13, year) + get_months_for_year(1, 10, year + 1)
-        )
-        if month < datetime.now()
-    ]
+    return [month.strftime(time_format) for month in (get_months_for_year(1, 13, year))]
 
 
 def get_months_for_year(start, end, year):

app/main/views/platform_admin.py

@@ -662,7 +662,21 @@ def create_global_stats(services):
         "email": {"delivered": 0, "failed": 0, "requested": 0},
         "sms": {"delivered": 0, "failed": 0, "requested": 0},
     }
+    # Issue #1323. The back end is now sending 'failure' instead of
+    # 'failed'.  Adjust it here, but keep it flexible in case
+    # the backend reverts to 'failed'.
     for service in services:
+        if service["statistics"]["sms"].get("failure") is not None:
+            service["statistics"]["sms"]["failed"] = service["statistics"]["sms"][
+                "failure"
+            ]
+        if service["statistics"]["email"].get("failure") is not None:
+            service["statistics"]["email"]["failed"] = service["statistics"]["email"][
+                "failure"
+            ]
+
+    for service in services:
+
         for msg_type, status in itertools.product(
             ("sms", "email"), ("delivered", "failed", "requested")
         ):

app/main/views/send.py

@@ -650,7 +650,9 @@ def check_messages(service_id, template_id, upload_id, row_index=2):
 @user_has_permissions("send_messages", restrict_admin_usage=True)
 def preview_job(service_id, template_id, upload_id, row_index=2):
     session["scheduled_for"] = request.form.get("scheduled_for", "")
-    data = _check_messages(service_id, template_id, upload_id, row_index, force_hide_sender=True)
+    data = _check_messages(
+        service_id, template_id, upload_id, row_index, force_hide_sender=True
+    )
 
     return render_template(
         "views/check/preview.html",
@@ -911,7 +913,9 @@ def preview_notification(service_id, template_id):
 
     return render_template(
         "views/notifications/preview.html",
-        **_check_notification(service_id, template_id, show_recipient=False, force_hide_sender=True),
+        **_check_notification(
+            service_id, template_id, show_recipient=False, force_hide_sender=True
+        ),
         scheduled_for=session["scheduled_for"],
         recipient=recipient,
     )

app/main/views/sign_in.py

@@ -6,6 +6,7 @@
 import requests
 from flask import (
     Markup,
+    Response,
     abort,
     current_app,
     flash,
@@ -26,6 +27,7 @@
 from app.models.user import InvitedUser, User
 from app.utils import hide_from_search_engines
 from app.utils.login import is_safe_redirect_url
+from app.utils.time import is_less_than_days_ago
 
 
 def _reformat_keystring(orig):
@@ -63,6 +65,10 @@ def _get_access_token(code, state):
     url = f"{base_url}{cli_assert}&{cli_assert_type}&{code_param}&grant_type=authorization_code"
     headers = {"Authorization": "Bearer %s" % token}
     response = requests.post(url, headers=headers)
+    if response.json().get("access_token") is None:
+        # Capture the response json here so it hopefully shows up in error reports
+        current_app.logger.error(f"Error when getting access token {response.json()}")
+        raise KeyError(f"'access_token' {response.json()}")
     access_token = response.json()["access_token"]
     return access_token
 
@@ -84,31 +90,59 @@ def _do_login_dot_gov():
     code = request.args.get("code")
     state = request.args.get("state")
     login_gov_error = request.args.get("error")
-    if code and state:
-        access_token = _get_access_token(code, state)
-        user_email, user_uuid = _get_user_email_and_uuid(access_token)
-        redirect_url = request.args.get("next")
+
+    if login_gov_error:
+        current_app.logger.error(f"login.gov error: {login_gov_error}")
+        raise Exception(f"Could not login with login.gov {login_gov_error}")
+    elif code and state:
 
         # activate the user
         try:
+            access_token = _get_access_token(code, state)
+            user_email, user_uuid = _get_user_email_and_uuid(access_token)
+            redirect_url = request.args.get("next")
             user = user_api_client.get_user_by_uuid_or_email(user_uuid, user_email)
-            activate_user(user["id"])
+
+            # Check if the email needs to be revalidated
+            is_fresh_email = is_less_than_days_ago(
+                user["email_access_validated_at"], 90
+            )
+            if not is_fresh_email:
+                return verify_email(user, redirect_url)
+
+            usr = User.from_email_address(user["email_address"])
+            activate_user(usr.id)
         except BaseException as be:  # noqa B036
             current_app.logger.error(be)
             error(401)
-
         return redirect(url_for("main.show_accounts_or_dashboard", next=redirect_url))
 
-    elif login_gov_error:
-        current_app.logger.error(f"login.gov error: {login_gov_error}")
-        raise Exception(f"Could not login with login.gov {login_gov_error}")
     # end login.gov
 
 
+def verify_email(user, redirect_url):
+    user_api_client.send_verify_code(user["id"], "email", None, redirect_url)
+    title = "Email resent" if request.args.get("email_resent") else "Check your email"
+    redirect_url = request.args.get("next")
+    return render_template(
+        "views/re-validate-email-sent.html", title=title, redirect_url=redirect_url
+    )
+
+
 @main.route("/sign-in", methods=(["GET", "POST"]))
 @hide_from_search_engines
 def sign_in():
-    _do_login_dot_gov()
+    # If we have to revalidated the email, send the message
+    # via email and redirect to the "verify your email page"
+    # and don't proceed further with login
+    email_verify_template = _do_login_dot_gov()
+    if (
+        email_verify_template
+        and not isinstance(email_verify_template, Response)
+        and "Check your email" in email_verify_template
+    ):
+        return email_verify_template
+
     redirect_url = request.args.get("next")
 
     if os.getenv("NOTIFY_E2E_TEST_EMAIL"):
@@ -192,7 +226,6 @@ def sign_in():
         form=form,
         again=bool(redirect_url),
         other_device=other_device,
-        login_gov_enabled=True,
         password_reset_url=password_reset_url,
         initial_signin_url=url,
     )

app/templates/error/500.html

@@ -7,7 +7,7 @@ <h1 class="heading-large">
         Sorry, we can't deliver what you asked for right now.
       </h1>
       <p class="usa-body">
-        Please try again later or <a class="usa-link" href="mailto:[email protected]"></a>email us</a> for more information.</p>
+        Please try again later or <a class="usa-link" href="mailto:[email protected]">email us</a> for more information.</p>
       </p>
     </div>
   </div>

app/templates/new/components/main_nav.html

@@ -0,0 +1,30 @@
+{% if help %}
+  {% include 'partials/tour.html' %}
+{% else %}
+<nav class="nav margin-bottom-4">
+  <a class="usa-button margin-top-1 margin-bottom-5 width-full"
+    href="{{ url_for('.choose_template', service_id=current_service.id) }}">Send messages</a>
+  <ul class="usa-sidenav">
+  {% if current_user.has_permissions() %}
+    {% if current_user.has_permissions('view_activity') %}
+      <li class="usa-sidenav__item"><a class="{{ main_navigation.is_selected('dashboard') }}" href="{{ url_for('.service_dashboard', service_id=current_service.id) }}">Dashboard</a></li>
+    {% endif %}
+    {% if not current_user.has_permissions('view_activity') %}
+      <li class="usa-sidenav__item"><a class="{{ casework_navigation.is_selected('sent-messages') }}" href="{{ url_for('.view_notifications', service_id=current_service.id, status='sending,delivered,failed') }}">Sent messages</a></li>
+    {% endif %}
+    {% if current_user.has_permissions('manage_service', allow_org_user=True) %}
+      {# <li class="usa-sidenav__item"><a class="{{ main_navigation.is_selected('usage') }}" href="{{ url_for('.usage', service_id=current_service.id) }}">Usage</a></li> #}
+    {% endif %}
+    <!-- {% if current_user.has_permissions('manage_api_keys', 'manage_service') %}
+      <li class="usa-sidenav__item"><a class="{{ main_navigation.is_selected('settings') }}" href="{{ url_for('.service_settings', service_id=current_service.id) }}">Settings</a></li>
+    {% endif %} -->
+    {% if current_user.has_permissions('manage_api_keys') %}
+      <!-- <li><a class="usa-link{{ main_navigation.is_selected('api-integration') }}" href="{{ url_for('.api_integration', service_id=current_service.id) }}">API integration</a></li> -->
+    {% endif %}
+  {% elif current_user.has_permissions(allow_org_user=True) %}
+    <li class="usa-sidenav__item"><a class="usa-link{{ main_navigation.is_selected('usage') }}" href="{{ url_for('.usage', service_id=current_service.id) }}">Usage</a></li>
+    <li class="usa-sidenav__item"><a class="usa-link{{ main_navigation.is_selected('team-members') }}" href="{{ url_for('.manage_users', service_id=current_service.id) }}">Team members</a></li>
+  {% endif %}
+  </ul>
+</nav>
+{% endif %}

app/templates/new/components/org_nav.html

@@ -0,0 +1,11 @@
+<nav class="navigation">
+  <ul>
+    <li><a class="usa-link{{ org_navigation.is_selected('dashboard') }}" href="{{ url_for('.organization_dashboard', org_id=current_org.id) }}">Usage</a></li>
+    <li><a class="usa-link{{ org_navigation.is_selected('team-members') }}" href="{{ url_for('.manage_org_users', org_id=current_org.id) }}">Team members</a></li>
+    {% if current_user.platform_admin %}
+    <li><a class="usa-link{{ org_navigation.is_selected('settings') }}" href="{{ url_for('.organization_settings', org_id=current_org.id) }}">Settings</a></li>
+    <li><a class="usa-link{{ org_navigation.is_selected('trial-services') }}" href="{{ url_for('.organization_trial_mode_services', org_id=current_org.id) }}">Trial mode services</a></li>
+    <li><a class="usa-link{{ org_navigation.is_selected('billing') }}" href="{{ url_for('.organization_billing', org_id=current_org.id) }}">Billing</a></li>
+    {% endif %}
+  </ul>
+</nav>

app/templates/new/components/service_navigation.html

@@ -0,0 +1,15 @@
+<div class="navigation-service margin-top-5 display-flex flex-align-end flex-justify border-bottom padding-bottom-1">
+  {% if current_service.organization_id %}
+    {% if current_user.platform_admin or
+      (current_user.belongs_to_organization(current_service.organization_id) and current_service.live) %}
+      <a href="{{ url_for('.organization_dashboard', org_id=current_service.organization_id) }}" class="usa-link navigation-organization-link">{{ current_service.organization_name }}</a>
+    {% endif %}
+  {% endif %}
+  <div class="font-body-2xl text-bold">
+    {{ current_service.name }}
+    {% if not current_service.active %}
+      <span class="navigation-service-name navigation-service-type--suspended">Suspended</span>
+    {% endif %}
+  </div>
+  <a href="{{ url_for('main.choose_account') }}" class="usa-link">Switch service</a>
+</div>