Merge branch 'dev'

.github/workflows/ai.yml

@@ -48,7 +48,7 @@ jobs:
         echo AZURE_OPENAI_MODEL=azure/gpt-4 >> .env
         echo OPENAI_API_VERSION=2024-02-01 >> .env
 
-        docker compose up -d --wait elasticsearch
+        docker compose up -d --wait opensearch
 
     - name: Run DeepEval Unit Tests
       run: |

.github/workflows/integration.yml

@@ -52,7 +52,7 @@ jobs:
         echo AZURE_OPENAI_API_KEY=${{ secrets.AZURE_OPENAI_API_KEY }} >> .env
         echo EMBEDDING_AZURE_OPENAI_ENDPOINT=${{ secrets.AZURE_OPENAI_ENDPOINT }} >> .env
         echo EMBEDDING_OPENAI_API_KEY=${{ secrets.AZURE_OPENAI_API_KEY }} >> .env
-        docker compose up -d --wait elasticsearch minio db
+        docker compose up -d --wait opensearch minio db
         docker compose up -d --wait django-app worker
         docker ps
 

.github/workflows/python-app.yml

@@ -43,7 +43,7 @@ jobs:
         mkdir -p data/elastic/
         chmod 777 data/elastic/
         cp tests/.env.test .env
-        docker compose up -d --wait elasticsearch
+        docker compose up -d --wait opensearch
 
     - name: Test redbox with pytest
       run: |
@@ -73,7 +73,7 @@ jobs:
         chmod -R 777 data/
         cp tests/.env.test .env
         sed -i -e "s/^OBJECT_STORE=.*/OBJECT_STORE=minio/" .env
-        docker compose up -d --wait elasticsearch minio db
+        docker compose up -d --wait opensearch minio db
         docker ps
 
     - name: Test django with pytest

Makefile

@@ -38,7 +38,8 @@ test-ai: ## Test code with live LLM
 
 .PHONY: test-redbox
 test-redbox: ## Test redbox
-	cd redbox-core && poetry install && poetry run pytest -m "not ai" --cov=redbox -v --cov-report=term-missing --cov-fail-under=60
+	cd redbox-core && PYTHONPATH=$(PWD)/django_app:$(PWD)/redbox-core DJANGO_SETTINGS_MODULE=django_app.settings poetry install && \
+	PYTHONPATH=$(PWD)/django_app:$(PWD)/redbox-core poetry run pytest -m "not ai" --cov=redbox -v --cov-report=term-missing --cov-fail-under=60
 
 .PHONY: test-django
 test-django: ## Test django-app
@@ -89,9 +90,9 @@ reset-db:  ## Reset Django database
 
 .PHONY: reset-elastic
 reset-elastic:  ## Reset Django database
-	docker compose down elasticsearch
+	docker compose down opensearch
 	rm -rf data/elastic/*
-	docker compose up -d elasticsearch --wait
+	docker compose up -d opensearch --wait
 
 .PHONY: docs-serve
 docs-serve:  ## Build and serve documentation

ai-tests/tests/test_ai.py

@@ -89,7 +89,7 @@ def test_usecases(test_case: AITestCase, loaded_docs: set[str], output_dir: Path
         with open(save_path, "w") as file:
             sys.stdout = file
             # Create and add a file handler for logging
-            file_handler = logging.StreamHandler(file)
+            logging.StreamHandler(file)
             # root_logger.addHandler(file_handler)
 
             # Remove other handlers temporarily if you want logging to go only to file

django_app/redbox_app/jinja2.py

@@ -3,7 +3,6 @@
 import humanize
 import jinja2
 import pytz
-import waffle
 from django.conf import settings
 from django.templatetags.static import static
 from django.urls import reverse

django_app/redbox_app/redbox_core/consumers.py

@@ -1,12 +1,12 @@
 import json
 import logging
 from asyncio import CancelledError
-from asgiref.sync import sync_to_async
 from collections import defaultdict
 from collections.abc import Mapping, Sequence
 from typing import Any, ClassVar
 from uuid import UUID
 
+from asgiref.sync import sync_to_async
 from channels.db import database_sync_to_async
 from channels.generic.websocket import AsyncWebsocketConsumer
 from django.conf import settings
@@ -66,7 +66,8 @@ def escape_curly_brackets(text: str):
 
 @sync_to_async
 def get_latest_complete_file(ref: str) -> File:
-    return File.objects.filter(original_file=ref, status=File.Status.complete).order_by('-created_at').first()
+    return File.objects.filter(original_file=ref, status=File.Status.complete).order_by("-created_at").first()
+
 
 class ChatConsumer(AsyncWebsocketConsumer):
     full_reply: ClassVar = []
@@ -352,7 +353,11 @@ async def handle_citations(self, citations: list[AICitation]):
                     # Use the async database query function
                     file = await get_latest_complete_file(s.source)
                     if file:
-                        payload = {"url": str(file.url), "file_name": file.file_name, "text_in_answer": c.text_in_answer}
+                        payload = {
+                            "url": str(file.url),
+                            "file_name": file.file_name,
+                            "text_in_answer": c.text_in_answer,
+                        }
                     else:
                         # If no file with Status.complete is found, handle it as None
                         payload = {"url": s.source, "file_name": s.source, "text_in_answer": c.text_in_answer}

django_app/redbox_app/redbox_core/forms.py

@@ -163,7 +163,16 @@ class Meta:
 class DemographicsForm(forms.ModelForm):
     class Meta:
         model = User
-        fields = ("name", "ai_experience", "business_unit", "role", "grade", "profession", "info_about_user", "redbox_response_preferences")
+        fields = (
+            "name",
+            "ai_experience",
+            "business_unit",
+            "role",
+            "grade",
+            "profession",
+            "info_about_user",
+            "redbox_response_preferences",
+        )
         labels: ClassVar[Mapping[str, str]] = {
             "name": "Full Name",
             "ai_experience": "How would you describe your level of experience with Generative AI tools?",

django_app/redbox_app/redbox_core/middleware.py

@@ -2,14 +2,15 @@
 
 from asgiref.sync import iscoroutinefunction
 from django.conf import settings
+from django.contrib.auth import get_user_model
 from django.http import HttpRequest, HttpResponse
 from django.utils.decorators import sync_and_async_middleware
 from rest_framework.authentication import BaseAuthentication
 from rest_framework.exceptions import AuthenticationFailed
-from django.contrib.auth import get_user_model
 
 User = get_user_model()
 
+
 @sync_and_async_middleware
 def nocache_middleware(get_response):
     if iscoroutinefunction(get_response):
@@ -81,18 +82,20 @@ def middleware(request: HttpRequest) -> HttpResponse:
 
     return middleware
 
+
 class APIKeyAuthentication(BaseAuthentication):
     def authenticate(self, request):
-        api_key = request.headers.get('X-API-KEY')
+        api_key = request.headers.get("X-API-KEY")
 
         if not api_key:
-            raise AuthenticationFailed('No API key provided')
+            msg = "No API key provided"
+            raise AuthenticationFailed(msg)
 
         if api_key == settings.REDBOX_API_KEY:
             user, _ = User.objects.get_or_create(
-                username='api_key_user',
-                defaults={'is_staff': True, 'is_superuser': False}
+                username="api_key_user", defaults={"is_staff": True, "is_superuser": False}
             )
             return (user, None)
 
-        raise AuthenticationFailed('Invalid API key')
+        msg = "Invalid API key"
+        raise AuthenticationFailed(msg)

django_app/redbox_app/redbox_core/models.py

@@ -9,15 +9,15 @@
 import jwt
 from django.conf import settings
 from django.contrib.auth.base_user import BaseUserManager as BaseSSOUserManager
+
+# from django_use_email_as_username.models import BaseUser, BaseUserManager
+from django.contrib.auth.models import AbstractBaseUser, Group, PermissionsMixin
 from django.contrib.postgres.fields import ArrayField
 from django.core import validators
 from django.db import models
 from django.db.models import Max, Min, Prefetch, UniqueConstraint
 from django.utils import timezone
 from django.utils.translation import gettext_lazy as _
-
-# from django_use_email_as_username.models import BaseUser, BaseUserManager
-from django.contrib.auth.models import AbstractBaseUser, Group, PermissionsMixin
 from yarl import URL
 
 from redbox.models.settings import get_settings
@@ -191,12 +191,13 @@ class SSOUserManager(BaseSSOUserManager):
     def _create_user(self, username, password, **extra_fields):
         """Create and save a User with the given email and password."""
         if not username:
-            raise ValueError("The given email must be set")
+            msg = "The given email must be set"
+            raise ValueError(msg)
         # email = self.normalize_email(email)
-        User = self.model(email=username, **extra_fields)
-        User.set_password(password)
-        User.save(using=self._db)
-        return User
+        user = self.model(email=username, **extra_fields)
+        user.set_password(password)
+        user.save(using=self._db)
+        return user
 
     def create_user(self, username, password=None, **extra_fields):
         """Create and save a regular User with the given email and password."""
@@ -210,9 +211,11 @@ def create_superuser(self, username, password=None, **extra_fields):
         extra_fields.setdefault("is_superuser", True)
 
         if extra_fields.get("is_staff") is not True:
-            raise ValueError("Superuser must have is_staff=True.")
+            msg = "Superuser must have is_staff=True."
+            raise ValueError(msg)
         if extra_fields.get("is_superuser") is not True:
-            raise ValueError("Superuser must have is_superuser=True.")
+            msg = "Superuser must have is_superuser=True."
+            raise ValueError(msg)
 
         return self._create_user(username, password, **extra_fields)
 
@@ -257,12 +260,24 @@ class Profession(models.TextChoices):
         OT = "OT", _("Other")
 
     class BusinessUnit(models.TextChoices):
-        COMPETITION_MARKETS_AND_REGULATORY_REFORM = "Competition, Markets and Regulatory Reform (CMRR)", _("Competition, Markets and Regulatory Reform (CMRR)")
+        COMPETITION_MARKETS_AND_REGULATORY_REFORM = (
+            "Competition, Markets and Regulatory Reform (CMRR)",
+            _("Competition, Markets and Regulatory Reform (CMRR)"),
+        )
         CORPORATE_SERVICES_GROUP = "Corporate Services Group (CSG)", _("Corporate Services Group (CSG)")
-        TRADE_POLICY_IMPLEMENTATION_AND_NEGOTIATIONS = "Trade Policy Implementation and Negotiations (TPIN)", _("Trade Policy Implementation and Negotiations (TPIN)")
-        ECONOMIC_SECURITY_AND_TRADE_RELATIONS = "Economic Security and Trade Relations (ESTR)", _("Economic Security and Trade Relations (ESTR)")
+        TRADE_POLICY_IMPLEMENTATION_AND_NEGOTIATIONS = (
+            "Trade Policy Implementation and Negotiations (TPIN)",
+            _("Trade Policy Implementation and Negotiations (TPIN)"),
+        )
+        ECONOMIC_SECURITY_AND_TRADE_RELATIONS = (
+            "Economic Security and Trade Relations (ESTR)",
+            _("Economic Security and Trade Relations (ESTR)"),
+        )
         STRATEGY_AND_INVESTMENT = "Strategy and Investment", _("Strategy and Investment")
-        DOMESTIC_AND_INTERNATIONAL_MARKETS_AND_EXPORTS_GROUP = "Domestic and International Markets and Exports Group (DIME) UK Teams", _("Domestic and International Markets and Exports Group (DIME) UK Teams")
+        DOMESTIC_AND_INTERNATIONAL_MARKETS_AND_EXPORTS_GROUP = (
+            "Domestic and International Markets and Exports Group (DIME) UK Teams",
+            _("Domestic and International Markets and Exports Group (DIME) UK Teams"),
+        )
         BUSINESS_GROUP = "Business Group", _("Business Group")
         OVERSEAS_REGIONS = "Overseas Regions", _("Overseas Regions")
         INDUSTRIAL_STRATEGY_UNIT = "Industrial Strategy Unit", _("Industrial Strategy Unit")

django_app/redbox_app/redbox_core/serializers.py

@@ -21,9 +21,7 @@ class Meta:
 class ChatMessageSerializer(serializers.ModelSerializer):
     selected_files = FileSerializer(many=True, read_only=True)
     source_files = FileSerializer(many=True, read_only=True)
-    token_use = ChatMessageTokenUseSerializer(
-        source="chatmessagetokenuse_set", many=True, read_only=True
-    )
+    token_use = ChatMessageTokenUseSerializer(source="chatmessagetokenuse_set", many=True, read_only=True)
 
     class Meta:
         model = ChatMessage
@@ -44,9 +42,7 @@ class Meta:
 
 
 class ChatSerializer(serializers.ModelSerializer):
-    messages = ChatMessageSerializer(
-        source="chatmessage_set", many=True, read_only=True
-    )
+    messages = ChatMessageSerializer(source="chatmessage_set", many=True, read_only=True)
 
     class Meta:
         model = Chat

django_app/redbox_app/redbox_core/views/__init__.py

@@ -1,5 +1,5 @@
 from redbox_app.redbox_core.dash_apps import report_app
-from redbox_app.redbox_core.views.api_views import user_view_pre_alpha
+from redbox_app.redbox_core.views.api_views import message_view_pre_alpha, user_view_pre_alpha
 from redbox_app.redbox_core.views.auth_views import sign_in_link_sent_view, sign_in_view, signed_out_view
 from redbox_app.redbox_core.views.chat_views import ChatsTitleView, ChatsView, DeleteChat, UpdateChatFeedback
 from redbox_app.redbox_core.views.citation_views import CitationsView
@@ -15,7 +15,7 @@
     remove_doc_view,
 )
 from redbox_app.redbox_core.views.info_views import accessibility_statement_view, privacy_notice_view, support_view
-from redbox_app.redbox_core.views.misc_views import SecurityTxtRedirectView, health, homepage_view, faq_view
+from redbox_app.redbox_core.views.misc_views import SecurityTxtRedirectView, faq_view, health, homepage_view
 from redbox_app.redbox_core.views.ratings_views import RatingsView
 from redbox_app.redbox_core.views.signup_views import Signup1, Signup2, Signup3, Signup4, Signup5, Signup6, Signup7
 
@@ -51,5 +51,6 @@
     "UpdateChatFeedback",
     "DeleteChat",
     "user_view_pre_alpha",
+    "message_view_pre_alpha",
     "faq_view",
 ]

django_app/redbox_app/redbox_core/views/api_views.py

@@ -1,18 +1,20 @@
 from django.contrib.auth import get_user_model
 from rest_framework.decorators import api_view, permission_classes
-from rest_framework.permissions import IsAuthenticated, IsAdminUser
 from rest_framework.pagination import PageNumberPagination
-from rest_framework.response import Response
+from rest_framework.permissions import IsAdminUser, IsAuthenticated
 
-from redbox_app.redbox_core.serializers import UserSerializer
+from redbox_app.redbox_core.models import ChatMessage
+from redbox_app.redbox_core.serializers import ChatMessageSerializer, UserSerializer
 
 User = get_user_model()
 
+
 class StandardResultsSetPagination(PageNumberPagination):
     page_size = 50
-    page_size_query_param = 'page_size'
+    page_size_query_param = "page_size"
     max_page_size = 100
 
+
 @api_view(["GET"])
 @permission_classes([IsAuthenticated, IsAdminUser])
 def user_view_pre_alpha(request):
@@ -21,4 +23,15 @@ def user_view_pre_alpha(request):
     queryset = User.objects.all()
     result_page = paginator.paginate_queryset(queryset, request)
     serializer = UserSerializer(result_page, many=True, read_only=True)
-    return paginator.get_paginated_response(serializer.data)
+    return paginator.get_paginated_response(serializer.data)
+
+
+@api_view(["GET"])
+@permission_classes([IsAuthenticated, IsAdminUser])
+def message_view_pre_alpha(request):
+    """Return paginated message data"""
+    paginator = StandardResultsSetPagination()
+    queryset = ChatMessage.objects.all()
+    result_page = paginator.paginate_queryset(queryset, request)
+    serializer = ChatMessageSerializer(result_page, many=True, read_only=True)
+    return paginator.get_paginated_response(serializer.data)

django_app/redbox_app/redbox_core/views/document_views.py

@@ -5,7 +5,7 @@
 
 from django.contrib.auth import get_user_model
 from django.contrib.auth.decorators import login_required
-from django.core.exceptions import FieldError, ValidationError, SuspiciousFileOperation
+from django.core.exceptions import FieldError, SuspiciousFileOperation, ValidationError
 from django.core.files.uploadedfile import UploadedFile
 from django.http import HttpRequest, HttpResponse, JsonResponse
 from django.shortcuts import get_object_or_404, redirect, render
@@ -78,7 +78,7 @@ def post(self, request: HttpRequest) -> HttpResponse:
         errors: MutableSequence[str] = []
 
         uploaded_files: MutableSequence[UploadedFile] = request.FILES.getlist("uploadDocs")
-        
+
         if not uploaded_files:
             errors.append("No document selected")
 
@@ -136,8 +136,11 @@ def ingest_file(uploaded_file: UploadedFile, user: User) -> Sequence[str]:
         except (ValueError, FieldError, ValidationError) as e:
             logger.exception("Error creating File model object for %s.", uploaded_file, exc_info=e)
             return e.args
-        except SuspiciousFileOperation as e:
-            return [f"Your file name is {len(uploaded_file.name)} characters long. The file name will need to be shortened by {len(uploaded_file.name) - 75} characters"]
+        except SuspiciousFileOperation:
+            return [
+                f"Your file name is {len(uploaded_file.name)} characters long. "
+                f"The file name will need to be shortened by {len(uploaded_file.name) - 75} characters"
+            ]
         else:
             async_task(ingest, file.id, task_name=file.unique_name, group="ingest")
 

django_app/redbox_app/redbox_core/views/info_views.py

@@ -3,7 +3,6 @@
 These shouldn't contain sensitive data and don't require login.
 """
 
-import waffle
 from django.conf import settings
 from django.shortcuts import render
 from django.views.decorators.http import require_http_methods