Merge pull request #2289 from GSA/notify-admin-2286

redis report

.ds.baseline

@@ -684,5 +684,5 @@
       }
     ]
   },
-  "generated_at": "2025-01-16T16:38:48Z"
+  "generated_at": "2025-02-03T17:01:06Z"
 }

.github/workflows/checks.yml

@@ -38,10 +38,10 @@ jobs:
           output: report-markdown
           annotations: failed-tests
           prnumber: ${{ steps.findPr.outputs.number }}
-      - name: Run style checks
-        run: poetry run flake8 .
       - name: Check imports alphabetized
         run: poetry run isort --check-only ./app ./tests
+      - name: Run style checks
+        run: poetry run flake8 .
       - name: Check dead code
         run: make dead-code
       - name: Run js tests

Makefile

@@ -62,6 +62,13 @@ py-lint: ## Run python linting scanners and black
 	poetry run flake8 .
 	poetry run isort --check-only ./app ./tests
 
+.PHONY: tada
+tada: ## Run python linting scanners and black
+	poetry run isort ./app ./tests
+	poetry run black .
+	poetry run flake8 .
+
+
 .PHONY: avg-complexity
 avg-complexity:
 	echo "*** Shows average complexity in radon of all code ***"

app/main/views/platform_admin.py

@@ -116,6 +116,70 @@ def download_all_users():
     return response
 
 
+@main.route("/platform-admin/get-redis-report")
+@user_is_platform_admin
+def get_redis_report():
+
+    memory_info = redis_client.info("memory")
+    memory_used = memory_info.get("used_memory_human", "N/A")
+    max_memory = memory_info.get("maxmemory_human", "N/A")
+    if max_memory == "0B":
+        max_memory = "No set limit"
+    mem_fragmentation = memory_info.get("mem_fragmentation_ratio", "N/A")
+    frag_quality = "Swapping (bad)"
+    if mem_fragmentation >= 1.0:
+        frag_quality = "Healthy"
+    if mem_fragmentation > 1.5:
+        frag_quality = "Problematic"
+    if mem_fragmentation > 2.0:
+        frag_quality = "Severe fragmentation"
+
+    frag_note = ""
+    if mem_fragmentation > 2.0:
+        frag_note = "Use MEMORY PURGE.\nReplace multiple small keys with hashes.\nAvoid long keys.\nSet max_memory."
+    elif mem_fragmentation < 1.0:
+        frag_note = "Allocate more RAM.\nSet max_memory."
+
+    keys = redis_client.keys("*")
+    key_details = []
+
+    for key in keys:
+        key_type = redis_client.type(key).decode("utf-8")
+        ttl = redis_client.ttl(key)
+        ttl_str = "No Expiry" if ttl == -1 else f"{ttl} seconds"
+        key_details.append(
+            {"Key": key.decode("utf-8"), "Type": key_type, "TTL": ttl_str}
+        )
+    output = StringIO()
+    writer = csv.writer(
+        output,
+    )
+    writer.writerow(["Redis Report"])
+    writer.writerow([])
+
+    writer.writerow(["Memory"])
+    writer.writerow(["", "Memory Used", memory_used])
+    writer.writerow(["", "Max Memory", max_memory])
+    writer.writerow(["", "Memory Fragmentation Ratio", mem_fragmentation])
+    writer.writerow(["", "Memory Fragmentation Quality", frag_quality, frag_note])
+    writer.writerow([])
+
+    writer.writerow(["Keys Overview"])
+    writer.writerow(["", "TTL", "Type", "Key"])
+    for key_detail in key_details:
+        writer.writerow(
+            ["", key_detail["TTL"], key_detail["Type"], key_detail["Key"][0:50]]
+        )
+
+    csv_data = output.getvalue()
+
+    # Create a direct download response with the CSV data and appropriate headers
+    response = Response(csv_data, content_type="text/csv; charset=utf-8")
+    response.headers["Content-Disposition"] = "attachment; filename=redis.csv"
+
+    return response
+
+
 def is_over_threshold(number, total, threshold):
     percentage = number / total * 100 if total else 0
     return percentage > threshold

app/main/views/sign_in.py

@@ -68,11 +68,12 @@ def _get_access_token(code):  # pragma: no cover
     id_token = get_id_token(response_json)
     nonce = id_token["nonce"]
     nonce_key = f"login-nonce-{unquote(nonce)}"
-    stored_nonce = redis_client.get(nonce_key).decode("utf8")
+    if not os.getenv("NOTIFY_ENVIRONMENT") == "development":
+        stored_nonce = redis_client.get(nonce_key).decode("utf8")
 
-    if nonce != stored_nonce:
-        current_app.logger.error(f"Nonce Error: {nonce} != {stored_nonce}")
-        abort(403)
+        if nonce != stored_nonce:
+            current_app.logger.error(f"Nonce Error: {nonce} != {stored_nonce}")
+            abort(403)
 
     try:
         access_token = response_json["access_token"]
@@ -112,7 +113,7 @@ def _do_login_dot_gov():  # $ pragma: no cover
         verify_key = f"login-verify_email-{unquote(state)}"
         verify_path = bool(redis_client.get(verify_key))
 
-        if not verify_path:
+        if not verify_path and not os.getenv("NOTIFY_ENVIRONMENT") == "development":
             state_key = f"login-state-{unquote(state)}"
             stored_state = unquote(redis_client.get(state_key).decode("utf8"))
             if state != stored_state:

app/templates/views/platform-admin/reports.html

@@ -34,5 +34,8 @@ <h1 class="font-body-2xl">
 <p>
   <a class="usa-link" href="{{ url_for('main.download_all_users') }}">Download All Users</a>
 </p>
+<p>
+  <a class="usa-link" href="{{ url_for('main.get_redis_report') }}">Get Redis Report</a>
+</p>
 
 {% endblock %}

notifications_utils/clients/redis/redis_client.py

@@ -149,6 +149,22 @@ def incr(self, key, raise_exception=False):
             except Exception as e:
                 self.__handle_exception(e, raise_exception, "incr", key)
 
+    def info(self, key):
+        if self.active:
+            return self.redis_store.info(key)
+
+    def keys(self, pattern):
+        if self.active:
+            return self.redis_store.keys(pattern)
+
+    def type(self, key):
+        if self.active:
+            return self.redis_store.type(key)
+
+    def ttl(self, key):
+        if self.active:
+            return self.redis_store.ttl(key)
+
     def get(self, key, raise_exception=False):
         key = prepare_value(key)
         if self.active:

tests/app/main/views/test_tour.py

@@ -174,6 +174,7 @@ def test_should_show_empty_text_box(
     # data-module=autofocus is set on a containing element so it
     # shouldn’t also be set on the textbox itself
     assert "data-module" not in textbox
+
     assert normalize_spaces(page.select_one("label[for=phone-number]").text) == "one"
 
 

tests/app/test_navigation.py

@@ -107,6 +107,7 @@
             "get_volumes_by_service",
             "get_example_csv",
             "get_notifications_as_json",
+            "get_redis_report",
             "get_started",
             "get_started_old",
             "go_to_dashboard_after_tour",