Detect if user requests to talk to a human (#118)

Co-authored-by: Sven Seeberg <[email protected]>

integreat_chat/chatanswers/services/answer.py

@@ -1,6 +1,7 @@
 """
 Retrieving matching documents for question an create summary text
 """
+
 import logging
 import asyncio
 import aiohttp
@@ -17,13 +18,14 @@
 from ..utils.rag_request import RagRequest
 from .llmapi import LlmApiClient, LlmMessage, LlmPrompt, LlmResponse
 
-LOGGER = logging.getLogger('django')
+LOGGER = logging.getLogger("django")
 
 
 class AnswerService:
     """
     Service for providing summary answers to question-like messages.
     """
+
     def __init__(self, rag_request: RagRequest) -> None:
         """
         param region: Integreat CMS region slug
@@ -35,20 +37,32 @@ def __init__(self, rag_request: RagRequest) -> None:
         self.llm_model_name = settings.RAG_MODEL
         self.llm_api = LlmApiClient()
 
-    def needs_answer(self, message: str) -> bool:
+    def skip_rag_answer(self, message: str, language_service: LanguageService) -> bool:
         """
         Check if a chat message is a question
 
         param message: a user message
         return: indication if the message needs an answer
         """
-        LOGGER.debug("Checking if message requires response.")
-        answer = self.llm_api.simple_prompt(Prompts.CHECK_QUESTION.format(message))
-        if answer.startswith("Yes"):
-            LOGGER.debug("Message requires response.")
-            return True
-        LOGGER.debug("Message does not require response.")
-        return False
+        LOGGER.debug("Checking if the user requests to talk to a human counselor")
+        if self.detect_request_human():
+            LOGGER.debug("User requests human intervention.")
+            message = Messages.TALK_TO_HUMAN
+        else:
+            answer = self.llm_api.simple_prompt(Prompts.CHECK_QUESTION.format(message))
+            if answer.startswith("Yes"):
+                LOGGER.debug("Message requires response.")
+                return None
+            message = Messages.NOT_QUESTION
+            LOGGER.debug("Message does not require response.")
+        return RagResponse(
+            [],
+            self.rag_request,
+            language_service.translate_message(
+                "en", self.language, message
+            ),
+            False,
+        )
 
     def get_documents(self) -> list:
         """
@@ -70,7 +84,7 @@ def get_documents(self) -> list:
         search_results = search.deduplicate_pages(
             search_results,
             settings.RAG_MAX_PAGES,
-            max_score=settings.RAG_DISTANCE_THRESHOLD
+            max_score=settings.RAG_DISTANCE_THRESHOLD,
         )
         LOGGER.debug("Number of retrieved documents: %i", len(search_results))
         if settings.RAG_RELEVANCE_CHECK:
@@ -87,26 +101,33 @@ def extract_answer(self) -> RagResponse:
         return: a dict containing a response and sources
         """
         question = str(self.rag_request)
+        language_service = LanguageService()
+
+        if response := self.skip_rag_answer(question, language_service):
+            return response
+
         LOGGER.debug("Retrieving documents.")
         documents = self.get_documents()
         LOGGER.debug("Retrieved %s documents.", len(documents))
 
-        context = "\n".join(
-            [result.content for result in documents]
-        )[:settings.RAG_CONTEXT_MAX_LENGTH]
+        context = "\n".join([result.content for result in documents])[
+            : settings.RAG_CONTEXT_MAX_LENGTH
+        ]
         if not documents:
-            language_service = LanguageService()
             return RagResponse(
                 documents,
                 self.rag_request,
                 language_service.translate_message(
-                    "en", self.language,
-                    Messages.NO_ANSWER
-                )
+                    "en", self.language, Messages.NO_ANSWER
+                ),
             )
         LOGGER.debug("Generating answer.")
-        answer = self.llm_api.simple_prompt(Prompts.RAG.format(self.language, question, context))
-        LOGGER.debug("Finished generating answer. Question: %s\nAnswer: %s", question, answer)
+        answer = self.llm_api.simple_prompt(
+            Prompts.RAG.format(self.language, question, context)
+        )
+        LOGGER.debug(
+            "Finished generating answer. Question: %s\nAnswer: %s", question, answer
+        )
         return RagResponse(documents, self.rag_request, answer)
 
     async def check_documents_relevance(self, question: str, search_results: list) -> bool:
@@ -135,3 +156,14 @@ async def check_documents_relevance(self, question: str, search_results: list) -
             if str(llm_response).startswith("yes"):
                 kept_documents.append(search_results[i])
         return kept_documents
+
+    def detect_request_human(self) -> bool:
+        """
+        Check if the user requests to talk to a human counselor or is asking a question
+        return: bool that indicates if the user requests a human or not
+        """
+        query = str(self.rag_request)
+        LOGGER.debug("Checking if user requests human intervention")
+        response = self.llm_api.simple_prompt(Prompts.HUMAN_REQUEST_CHECK.format(query))
+        LOGGER.debug("Finished checking if user requests human. Response: %s", response)
+        return response.lower().startswith("yes")

integreat_chat/chatanswers/static/messages.py

@@ -5,3 +5,7 @@
 class Messages:
 
     NO_ANSWER = """Sorry, we could not find an answer for you in the Integreat content. Please wait for a message from a human advisor."""
+
+    TALK_TO_HUMAN = """We will forward your request to a human advisor. Please wait."""
+
+    NOT_QUESTION = """Sorry, kindly re-phrase your message as a question you would like to ask."""

integreat_chat/chatanswers/static/prompts.py

@@ -40,3 +40,8 @@ class Prompts:
     OPTIMIZE_MESSAGE = """Please summarize the following text into one terse sentence or question. Only answer with the summary, no text around it.
     
 Text: {0}"""
+
+    HUMAN_REQUEST_CHECK = """You are an assistant trained to classify user intent. Your task is to determine whether the user wants to talk to a human.
+Always respond with "Yes" if the user is asking for human assistance, and "No otherwise"
+
+User query: {0}"""

integreat_chat/chatanswers/utils/rag_request.py

@@ -1,6 +1,7 @@
 """
 Message for processing a user message / RAG request
 """
+
 import logging
 
 from django.conf import settings
@@ -9,7 +10,7 @@
 from integreat_chat.core.utils.integreat_request import IntegreatRequest
 from integreat_chat.chatanswers.services.query_transformer import QueryTransformer
 
-LOGGER = logging.getLogger('django')
+LOGGER = logging.getLogger("django")
 
 
 class RagRequest(IntegreatRequest):
@@ -52,5 +53,5 @@ def as_dict(self) -> dict:
             "message": str(self),
             "rag_language": self.use_language,
             "gui_language": self.gui_language,
-            "region": self.region
+            "region": self.region,
         }

integreat_chat/chatanswers/utils/rag_response.py

@@ -1,19 +1,29 @@
 """
 RAG response
 """
+
 from django.conf import settings
 
 from integreat_chat.search.utils.search_response import Document
 from integreat_chat.core.utils.integreat_request import IntegreatRequest
 
-class RagResponse():
+
+class RagResponse:
     """
     Representation of RAG response
     """
-    def __init__(self, documents: list[Document], request: IntegreatRequest, rag_response: str):
+
+    def __init__(
+        self,
+        documents: list[Document],
+        request: IntegreatRequest,
+        rag_response: str,
+        automatic_answers: bool = True,
+    ):
         self.documents = documents
         self.request = request
         self.rag_response = rag_response
+        self.automatic_answers = automatic_answers
 
     def __str__(self):
         """
@@ -22,9 +32,7 @@ def __str__(self):
         """
         if self.request.gui_language != self.request.use_language:
             message = self.request.language_service.translate_message(
-                self.request.use_language,
-                self.request.gui_language,
-                self.rag_response
+                self.request.use_language, self.request.gui_language, self.rag_response
             )
         else:
             message = self.rag_response
@@ -37,14 +45,17 @@ def create_citation(self):
         sources = []
         for document in self.documents:
             if self.request.gui_language != self.request.use_language:
-                sources.append((document.get_source_for_language(self.request.gui_language)))
+                sources.append(
+                    (document.get_source_for_language(self.request.gui_language))
+                )
             else:
                 sources.append((document.chunk_source_path, document.title))
 
         citation = "".join(
             [
                 f"<li><a href='https://{settings.INTEGREAT_APP_DOMAIN}{path}'>{title}</a></li>"
-                for path, title in sources if title is not None
+                for path, title in sources
+                if title is not None
             ]
         )
         return f"\n<ul>{citation}</ul>" if citation else ""
@@ -60,9 +71,13 @@ def as_dict(self):
             "rag_language": self.request.use_language,
             "rag_message": self.request.translated_message,
             "rag_sources": [document.chunk_source_path for document in self.documents],
-            "details": [{
-                "source": document.chunk_source_path,
-                "score": document.score,
-                "context": document.content,
-            } for document in self.documents],
+            "automatic_answers": self.automatic_answers,
+            "details": [
+                {
+                    "source": document.chunk_source_path,
+                    "score": document.score,
+                    "context": document.content,
+                }
+                for document in self.documents
+            ],
         }

integreat_chat/chatanswers/views.py

@@ -21,7 +21,7 @@ def extract_answer(request):
     Extract an answer for a user query from Integreat content. Expects a JSON body with message
     and language attributes
     """
-    rag_response = {}
+    rag_response = {"status": "error"}
     if (
         request.method in ("POST")
         and request.META.get("CONTENT_TYPE").lower() == "application/json"