02/04/2025 Production Deploy

.ds.baseline

@@ -277,15 +277,15 @@
         "filename": "tests/app/notifications/test_receive_notification.py",
         "hashed_secret": "913a73b565c8e2c8ed94497580f619397709b8b6",
         "is_verified": false,
-        "line_number": 26,
+        "line_number": 27,
         "is_secret": false
       },
       {
         "type": "Base64 High Entropy String",
         "filename": "tests/app/notifications/test_receive_notification.py",
         "hashed_secret": "d70eab08607a4d05faa2d0d6647206599e9abc65",
         "is_verified": false,
-        "line_number": 56,
+        "line_number": 57,
         "is_secret": false
       }
     ],
@@ -295,7 +295,7 @@
         "filename": "tests/app/notifications/test_validators.py",
         "hashed_secret": "6c1a8443963d02d13ffe575a71abe19ea731fb66",
         "is_verified": false,
-        "line_number": 768,
+        "line_number": 672,
         "is_secret": false
       }
     ],
@@ -305,7 +305,7 @@
         "filename": "tests/app/service/test_rest.py",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 1284,
+        "line_number": 1285,
         "is_secret": false
       }
     ],
@@ -341,15 +341,15 @@
         "filename": "tests/app/user/test_rest.py",
         "hashed_secret": "5baa61e4c9b93f3f0682250b6cf8331b7ee68fd8",
         "is_verified": false,
-        "line_number": 108,
+        "line_number": 110,
         "is_secret": false
       },
       {
         "type": "Secret Keyword",
         "filename": "tests/app/user/test_rest.py",
         "hashed_secret": "0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33",
         "is_verified": false,
-        "line_number": 826,
+        "line_number": 864,
         "is_secret": false
       }
     ],
@@ -384,5 +384,5 @@
       }
     ]
   },
-  "generated_at": "2024-10-31T21:25:32Z"
+  "generated_at": "2025-02-10T16:57:15Z"
 }

.github/workflows/checks.yml

@@ -47,10 +47,11 @@ jobs:
         NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_PASSWORD }}
         NOTIFY_E2E_TEST_HTTP_AUTH_USER: ${{ secrets.NOTIFY_E2E_TEST_HTTP_AUTH_USER }}
         NOTIFY_E2E_TEST_PASSWORD: ${{ secrets.NOTIFY_E2E_TEST_PASSWORD }}
-    - name: Run style checks
-      run: poetry run flake8 .
+
     - name: Check imports alphabetized
       run: poetry run isort --check-only ./app ./tests
+    - name: Run style checks
+      run: poetry run flake8 .
     - name: Check for dead code
       run: make dead-code
     - name: Run tests with coverage
@@ -136,7 +137,7 @@ jobs:
         env:
           SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
       - name: Run OWASP API Scan
-        uses: zaproxy/action-api-scan@v0.5.0
+        uses: zaproxy/action-api-scan@v0.9.0
         with:
           docker_name: 'ghcr.io/zaproxy/zaproxy:weekly'
           target: 'http://localhost:6011/docs/openapi.yml'

.github/workflows/daily_checks.yml

@@ -84,7 +84,7 @@ jobs:
         env:
           SQLALCHEMY_DATABASE_TEST_URI: postgresql://user:password@localhost:5432/test_notification_api
       - name: Run OWASP API Scan
-        uses: zaproxy/action-api-scan@v0.5.0
+        uses: zaproxy/action-api-scan@v0.9.0
         with:
           docker_name: 'ghcr.io/zaproxy/zaproxy:weekly'
           target: 'http://localhost:6011/docs/openapi.yml'

Makefile

@@ -31,6 +31,14 @@ bootstrap-with-docker: ## Build the image to run the app in Docker
 run-procfile:
 	poetry run honcho start -f Procfile.dev
 
+
+
+.PHONY: tada
+tada:
+	poetry run isort .
+	poetry run black .
+	poetry run flake8 .
+
 .PHONY: avg-complexity
 avg-complexity:
 	echo "*** Shows average complexity in radon of all code ***"
@@ -83,7 +91,7 @@ test: export NEW_RELIC_ENVIRONMENT=test
 test: ## Run tests and create coverage report
 	poetry run black .
 	poetry run flake8 .
-	poetry run isort --check-only ./app ./tests
+	poetry run isort ./app ./tests
 	poetry run coverage run --omit=*/migrations/*,*/tests/* -m pytest --maxfail=10
 
     ## TODO set this back to 95 asap

app/__init__.py

@@ -286,6 +286,13 @@ def record_request_details():
     @app.after_request
     def after_request(response):
         response.headers.add("X-Content-Type-Options", "nosniff")
+
+        # Some dynamic scan findings
+        response.headers.add("Cross-Origin-Opener-Policy", "same-origin")
+        response.headers.add("Cross-Origin-Embedder-Policy", "require-corp")
+        response.headers.add("Cross-Origin-Resource-Policy", "same-origin")
+        response.headers.add("Cross-Origin-Opener-Policy", "same-origin")
+
         return response
 
     @app.errorhandler(Exception)

app/billing/rest.py

@@ -1,5 +1,6 @@
 from flask import Blueprint, jsonify, request
 
+from app import db
 from app.billing.billing_schemas import (
     create_or_update_free_sms_fragment_limit_schema,
     serialize_ft_billing_remove_emails,
@@ -60,7 +61,7 @@ def get_free_sms_fragment_limit(service_id):
     )
 
     if annual_billing is None:
-        service = Service.query.get(service_id)
+        service = db.session.get(Service, service_id)
         # An entry does not exist in annual_billing table for that service and year.
         # Set the annual billing to the default free allowance based on the organization type of the service.
 

app/celery/nightly_tasks.py

@@ -52,7 +52,8 @@ def cleanup_unfinished_jobs():
         # The query already checks that the processing_finished time is null, so here we are saying
         # if it started more than 4 hours ago, that's too long
         try:
-            acceptable_finish_time = job.processing_started + timedelta(minutes=5)
+            if job.processing_started is not None:
+                acceptable_finish_time = job.processing_started + timedelta(minutes=5)
         except TypeError:
             current_app.logger.exception(
                 f"Job ID {job.id} processing_started is {job.processing_started}.",

app/celery/provider_tasks.py

@@ -14,6 +14,7 @@
 from app.delivery import send_to_providers
 from app.enums import NotificationStatus
 from app.exceptions import NotificationTechnicalFailureException
+from notifications_utils.clients.redis import total_limit_cache_key
 
 
 @notify_celery.task(
@@ -41,6 +42,9 @@ def deliver_sms(self, notification_id):
         # Code branches off to send_to_providers.py
         send_to_providers.send_sms_to_provider(notification)
 
+        cache_key = total_limit_cache_key(notification.service_id)
+        redis_store.incr(cache_key)
+
     except Exception as e:
         update_notification_status_by_id(
             notification_id,

app/celery/scheduled_tasks.py

@@ -2,10 +2,10 @@
 from datetime import datetime, timedelta
 
 from flask import current_app
-from sqlalchemy import between
+from sqlalchemy import between, select, union
 from sqlalchemy.exc import SQLAlchemyError
 
-from app import notify_celery, redis_store, zendesk_client
+from app import db, notify_celery, redis_store, zendesk_client
 from app.celery.tasks import (
     get_recipient_csv_and_template_and_sender_id,
     process_incomplete_jobs,
@@ -20,7 +20,7 @@
 from app.dao.invited_user_dao import expire_invitations_created_more_than_two_days_ago
 from app.dao.jobs_dao import (
     dao_set_scheduled_jobs_to_pending,
-    dao_update_job,
+    dao_update_job_status_to_error,
     find_jobs_with_missing_rows,
     find_missing_row_for_job,
 )
@@ -115,30 +115,34 @@ def check_job_status():
     end_minutes_ago = utc_now() - timedelta(minutes=END_MINUTES)
     start_minutes_ago = utc_now() - timedelta(minutes=START_MINUTES)
 
-    incomplete_in_progress_jobs = Job.query.filter(
+    incomplete_in_progress_jobs = select(Job).where(
         Job.job_status == JobStatus.IN_PROGRESS,
         between(Job.processing_started, start_minutes_ago, end_minutes_ago),
     )
-    incomplete_pending_jobs = Job.query.filter(
+    incomplete_pending_jobs = select(Job).where(
         Job.job_status == JobStatus.PENDING,
         Job.scheduled_for.isnot(None),
         between(Job.scheduled_for, start_minutes_ago, end_minutes_ago),
     )
-
-    jobs_not_complete_after_allotted_time = (
-        incomplete_in_progress_jobs.union(incomplete_pending_jobs)
-        .order_by(Job.processing_started, Job.scheduled_for)
-        .all()
+    jobs_not_completed_after_allotted_time = union(
+        incomplete_in_progress_jobs, incomplete_pending_jobs
+    )
+    jobs_not_completed_after_allotted_time = (
+        jobs_not_completed_after_allotted_time.order_by(
+            Job.processing_started, Job.scheduled_for
+        )
     )
 
+    jobs_not_complete_after_allotted_time = db.session.execute(
+        jobs_not_completed_after_allotted_time
+    ).all()
+
     # temporarily mark them as ERROR so that they don't get picked up by future check_job_status tasks
     # if they haven't been re-processed in time.
     job_ids = []
     for job in jobs_not_complete_after_allotted_time:
-        job.job_status = JobStatus.ERROR
-        dao_update_job(job)
+        dao_update_job_status_to_error(job)
         job_ids.append(str(job.id))
-
     if job_ids:
         current_app.logger.info("Job(s) {} have not completed.".format(job_ids))
         process_incomplete_jobs.apply_async([job_ids], queue=QueueNames.JOBS)
@@ -168,6 +172,7 @@ def replay_created_notifications():
 
 @notify_celery.task(name="check-for-missing-rows-in-completed-jobs")
 def check_for_missing_rows_in_completed_jobs():
+
     jobs = find_jobs_with_missing_rows()
     for job in jobs:
         (

app/celery/tasks.py

@@ -159,7 +159,14 @@ def process_row(row, template, job, service, sender_id=None):
     return notification_id
 
 
+# TODO
+# Originally this was checking a daily limit
+# It is now checking an overall limit (annual?) for the free tier
+# Is there any limit for the paid tier?
+# Assuming the limit is annual, is it calendar year, fiscal year, MOU year?
+# Do we need a command to run to clear the redis value, or should it happen automatically?
 def __total_sending_limits_for_job_exceeded(service, job, job_id):
+    print(hilite("ENTER __total_sending_limits_for_job_exceeded"))
     try:
         total_sent = check_service_over_total_message_limit(KeyType.NORMAL, service)
         if total_sent + job.notification_count > service.total_message_limit:
@@ -172,7 +179,7 @@ def __total_sending_limits_for_job_exceeded(service, job, job_id):
         dao_update_job(job)
         current_app.logger.exception(
             "Job {} size {} error. Total sending limits {} exceeded".format(
-                job_id, job.notification_count, service.message_limit
+                job_id, job.notification_count, service.total_message_limit
             ),
         )
         return True

app/commands.py

@@ -656,7 +656,7 @@ def populate_annual_billing_with_defaults(year, missing_services_only):
                     AnnualBilling.financial_year_start == year,
                 ),
             )
-            .filter(AnnualBilling.id == None)  # noqa
+            .where(AnnualBilling.id == None)  # noqa
         )
         active_services = db.session.execute(stmt).scalars().all()
     else:
@@ -665,7 +665,7 @@ def populate_annual_billing_with_defaults(year, missing_services_only):
     previous_year = year - 1
     services_with_zero_free_allowance = (
         db.session.query(AnnualBilling.service_id)
-        .filter(
+        .where(
             AnnualBilling.financial_year_start == previous_year,
             AnnualBilling.free_sms_fragment_limit == 0,
         )
@@ -846,6 +846,19 @@ def create_new_service(name, message_limit, restricted, email_from, created_by_i
         db.session.rollback()
 
 
+@notify_command(name="get-service-sender-phones")
+@click.option("-s", "--service_id", required=True, prompt=True)
+def get_service_sender_phones(service_id):
+    sender_phone_numbers = """
+            select sms_sender, is_default
+            from service_sms_senders
+            where service_id = :service_id
+        """
+    rows = db.session.execute(text(sender_phone_numbers), {"service_id": service_id})
+    for row in rows:
+        print(row)
+
+
 @notify_command(name="promote-user-to-platform-admin")
 @click.option("-u", "--user-email-address", required=True, prompt=True)
 def promote_user_to_platform_admin(user_email_address):

app/config.py

@@ -2,10 +2,12 @@
 from datetime import datetime, timedelta
 from os import getenv, path
 
+from boto3 import Session
 from celery.schedules import crontab
 from kombu import Exchange, Queue
 
 import notifications_utils
+from app.clients import AWS_CLIENT_CONFIG
 from app.cloudfoundry_config import cloud_config
 
 
@@ -51,6 +53,13 @@ class TaskNames(object):
     SCAN_FILE = "scan-file"
 
 
+session = Session(
+    aws_access_key_id=getenv("CSV_AWS_ACCESS_KEY_ID"),
+    aws_secret_access_key=getenv("CSV_AWS_SECRET_ACCESS_KEY"),
+    region_name=getenv("CSV_AWS_REGION"),
+)
+
+
 class Config(object):
     NOTIFY_APP_NAME = "api"
     DEFAULT_REDIS_EXPIRE_TIME = 4 * 24 * 60 * 60
@@ -166,6 +175,9 @@ class Config(object):
 
     current_minute = (datetime.now().minute + 1) % 60
 
+    S3_CLIENT = session.client("s3")
+    S3_RESOURCE = session.resource("s3", config=AWS_CLIENT_CONFIG)
+
     CELERY = {
         "worker_max_tasks_per_child": 500,
         "task_ignore_result": True,
@@ -327,7 +339,7 @@ class Config(object):
 
     FREE_SMS_TIER_FRAGMENT_COUNT = 250000
 
-    TOTAL_MESSAGE_LIMIT = 250000
+    TOTAL_MESSAGE_LIMIT = 100000
 
     DAILY_MESSAGE_LIMIT = notifications_utils.DAILY_MESSAGE_LIMIT