Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Deploy to production #233

Merged
merged 3 commits into from
Jul 22, 2024
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion chat/src/handlers/chat.py
Original file line number Diff line number Diff line change
@@ -13,7 +13,7 @@
RESPONSE_TYPES = {
"base": ["answer", "ref"],
"debug": ["answer", "attributes", "azure_endpoint", "deployment_name", "is_superuser", "k", "openai_api_version", "prompt", "question", "ref", "temperature", "text_key", "token_counts"],
"log": ["answer", "is_superuser", "k", "openai_api_version", "prompt", "question", "ref", "temperature", "token_counts"]
"log": ["answer", "deployment_name", "is_superuser", "k", "openai_api_version", "prompt", "question", "ref", "source_documents", "temperature", "token_counts"]
}

def handler(event, context):
8 changes: 4 additions & 4 deletions chat/src/handlers/opensearch_neural_search.py
Original file line number Diff line number Diff line change
@@ -28,19 +28,19 @@ def __init__(
self.text_field = text_field

def similarity_search(
self, query: str, k: int = 10, subquery: Any = None, **kwargs: Any
self, query: str, k: int = 10, **kwargs: Any
) -> List[Document]:
"""Return docs most similar to the embedding vector."""
docs_with_scores = self.similarity_search_with_score(
query, k, subquery, **kwargs
query, k, **kwargs
)
return [doc[0] for doc in docs_with_scores]

def similarity_search_with_score(
self, query: str, k: int = 10, subquery: Any = None, **kwargs: Any
self, query: str, k: int = 10, **kwargs: Any
) -> List[Tuple[Document, float]]:
"""Return docs most similar to query."""
dsl = hybrid_query(query=query, model_id=self.model_id, vector_field=self.vector_field, k=k, subquery=subquery, **kwargs)
dsl = hybrid_query(query=query, model_id=self.model_id, vector_field=self.vector_field, k=k, **kwargs)
response = self.client.search(index=self.index, body=dsl, params={"search_pipeline": self.search_pipeline} if self.search_pipeline else None)
documents_with_scores = [
(
23 changes: 7 additions & 16 deletions chat/src/helpers/hybrid_query.py
Original file line number Diff line number Diff line change
@@ -11,22 +11,18 @@ def filter(query: dict):
}
}

def hybrid_query(query: str, model_id: str, vector_field: str = "embedding", k: int = 10, subquery: Any = None, **kwargs: Any):
if subquery:
weights = [0.5, 0.3, 0.2]
else:
weights = [0.7, 0.3]

def hybrid_query(query: str, model_id: str, vector_field: str = "embedding", k: int = 10, **kwargs: Any):
result = {
"size": k,
"query": {
"hybrid": {
"queries": [
filter({
"query_string": {
"default_operator": "AND",
"fields": ["title^5", "all_controlled_labels", "all_ids^5"],
"query": query
"default_operator": "AND",
"fields": ["all_titles^5", "all_controlled_labels", "all_ids^5"],
"query": query,
"analyzer": "english"
}
}),
filter({
@@ -47,7 +43,7 @@ def hybrid_query(query: str, model_id: str, vector_field: str = "embedding", k:
"normalization-processor": {
"combination": {
"parameters": {
"weights": weights
"weights": [0.25, 0.75]
},
"technique": "arithmetic_mean"
},
@@ -60,12 +56,7 @@ def hybrid_query(query: str, model_id: str, vector_field: str = "embedding", k:
}
}

if subquery:
result["query"]["hybrid"]["queries"].append(filter(subquery))

for key, value in kwargs.items():
result[key] = value

return result


19 changes: 19 additions & 0 deletions chat/src/helpers/metrics.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,24 @@
import tiktoken

def debug_response(config, response, original_question):
source_urls = [doc["api_link"] for doc in original_question.get("source_documents", [])]

return {
"answer": response,
"attributes": config.attributes,
"azure_endpoint": config.azure_endpoint,
"deployment_name": config.deployment_name,
"is_superuser": config.api_token.is_superuser(),
"k": config.k,
"openai_api_version": config.openai_api_version,
"prompt": config.prompt_text,
"question": config.question,
"ref": config.ref,
"source_documents": source_urls,
"temperature": config.temperature,
"text_key": config.text_key,
"token_counts": token_usage(config, response, original_question),
}

def token_usage(config, response, original_question):
data = {
30 changes: 2 additions & 28 deletions chat/src/helpers/response.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from helpers.metrics import token_usage
from helpers.metrics import debug_response
from langchain_core.output_parsers import StrOutputParser
from langchain_core.runnables import RunnableLambda, RunnablePassthrough

@@ -16,23 +16,6 @@ def __init__(self, config):
self.store = {}

def debug_response_passthrough(self):
def debug_response(config, response, original_question):
return {
"answer": response,
"attributes": config.attributes,
"azure_endpoint": config.azure_endpoint,
"deployment_name": config.deployment_name,
"is_superuser": config.api_token.is_superuser(),
"k": config.k,
"openai_api_version": config.openai_api_version,
"prompt": config.prompt_text,
"question": config.question,
"ref": config.ref,
"temperature": config.temperature,
"text_key": config.text_key,
"token_counts": token_usage(config, response, original_question),
}

return RunnableLambda(lambda x: debug_response(self.config, x, self.original_question))

def original_question_passthrough(self):
@@ -56,16 +39,7 @@ def get_and_send_original_question(docs):

def prepare_response(self):
try:
subquery = {
"match": {
"all_titles": {
"query": self.config.question,
"operator": "AND",
"analyzer": "english"
}
}
}
retriever = self.config.opensearch.as_retriever(search_type="similarity", search_kwargs={"k": self.config.k, "subquery": subquery, "_source": {"excludes": ["embedding"]}})
retriever = self.config.opensearch.as_retriever(search_type="similarity", search_kwargs={"k": self.config.k, "_source": {"excludes": ["embedding"]}})
chain = (
{"context": retriever, "question": RunnablePassthrough()}
| self.original_question_passthrough()
10 changes: 4 additions & 6 deletions chat/test/helpers/test_hybrid_query.py
Original file line number Diff line number Diff line change
@@ -6,19 +6,17 @@

class TestFunction(TestCase):
def test_hybrid_query(self):
subquery = { "term": { "title": { "value": "The Title" } } }
dsl = hybrid_query("Question?", "MODEL_ID", k=10, subquery=subquery)
dsl = hybrid_query("Question?", "MODEL_ID", k=10)
subject = dsl["query"]["hybrid"]["queries"]

checks = [
(lambda x: x["query_string"]["query"], "Question?"),
(lambda x: x["neural"]["embedding"]["model_id"], "MODEL_ID"),
(lambda x: x["term"]["title"]["value"], "The Title")
(lambda x: x["neural"]["embedding"]["model_id"], "MODEL_ID")
]

self.assertEqual(len(subject), 3)
self.assertEqual(len(subject), 2)

for i in range(3):
for i in range(2):
lookup, expected = checks[i]
queries = subject[i]["bool"]["must"]
self.assertEqual(lookup(queries[0]), expected)
95 changes: 72 additions & 23 deletions chat/test/helpers/test_metrics.py
Original file line number Diff line number Diff line change
@@ -5,53 +5,102 @@
sys.path.append('./src')

from unittest import TestCase, mock
from helpers.metrics import count_tokens, token_usage
from helpers.metrics import count_tokens, debug_response, token_usage
from event_config import EventConfig



@mock.patch.dict(
os.environ,
{
"AZURE_OPENAI_RESOURCE_NAME": "test",
"WEAVIATE_URL": "http://test",
"WEAVIATE_API_KEY": "test"
},
)
class TestMetrics(TestCase):
def test_token_usage(self):
original_question = {
"question": "What is your name?",
"source_documents": [],
@mock.patch.dict(
os.environ,
{
"AZURE_OPENAI_RESOURCE_NAME": "test",
"WEAVIATE_URL": "http://test",
"WEAVIATE_API_KEY": "test"
},
)
def setUp(self):
self.question = "What is your name?"
self.original_question = {
"question": self.question,
"source_documents": [
{
"accession_number": "SourceDoc:1",
"api_link": "https://api.dc.library.northwestern.edu/api/v2/works/881e8cae-67be-4e04-9970-7eafb52b2c5c",
"canonical_link": "https://dc.library.northwestern.edu/items/881e8cae-67be-4e04-9970-7eafb52b2c5c",
"title": "Source Document One!"
},
{
"accession_number": "SourceDoc:2",
"api_link": "https://api.dc.library.northwestern.edu/api/v2/works/ac0b2a0d-8f80-420a-b1a1-63b6ac2299f1",
"canonical_link": "https://dc.library.northwestern.edu/items/ac0b2a0d-8f80-420a-b1a1-63b6ac2299f1",
"title": "Source Document Two!"
},
{
"accession_number": "SourceDoc:3",
"api_link": "https://api.dc.library.northwestern.edu/api/v2/works/11569bb5-1b89-4fa9-bdfb-2caf2ded5aa5",
"canonical_link": "https://dc.library.northwestern.edu/items/11569bb5-1b89-4fa9-bdfb-2caf2ded5aa5",
"title": "Source Document Three!"
},
{
"accession_number": "SourceDoc:4",
"api_link": "https://api.dc.library.northwestern.edu/api/v2/works/211eeeca-d56e-4c6e-9123-1612d72258f9",
"canonical_link": "https://dc.library.northwestern.edu/items/211eeeca-d56e-4c6e-9123-1612d72258f9",
"title": "Source Document Four!"
},
{
"accession_number": "SourceDoc:5",
"api_link": "https://api.dc.library.northwestern.edu/api/v2/works/10e45e7a-8011-4ac5-97df-efa6a5439d0e",
"canonical_link": "https://dc.library.northwestern.edu/items/10e45e7a-8011-4ac5-97df-efa6a5439d0e",
"title": "Source Document Five!"
}
],
}
event = {
self.event = {
"body": json.dumps({
"deployment_name": "test",
"index": "test",
"k": 1,
"k": 5,
"openai_api_version": "2019-05-06",
"prompt": "This is a test prompt.",
"question": original_question,
"question": self.question,
"ref": "test",
"temperature": 0.5,
"text_key": "text",
"auth": "test123"
})
}
config = EventConfig(event=event)

response = {
self.config = EventConfig(event=self.event)
self.response = {
"output_text": "This is a test response.",
}

def test_debug_response(self):
result = debug_response(self.config, self.response, self.original_question)

self.assertEqual(result["k"], 5)
self.assertEqual(result["question"], self.question)
self.assertEqual(result["ref"], "test")
self.assertEqual(
result["source_documents"],
[
"https://api.dc.library.northwestern.edu/api/v2/works/881e8cae-67be-4e04-9970-7eafb52b2c5c",
"https://api.dc.library.northwestern.edu/api/v2/works/ac0b2a0d-8f80-420a-b1a1-63b6ac2299f1",
"https://api.dc.library.northwestern.edu/api/v2/works/11569bb5-1b89-4fa9-bdfb-2caf2ded5aa5",
"https://api.dc.library.northwestern.edu/api/v2/works/211eeeca-d56e-4c6e-9123-1612d72258f9",
"https://api.dc.library.northwestern.edu/api/v2/works/10e45e7a-8011-4ac5-97df-efa6a5439d0e"
]
)

result = token_usage(config, response, original_question)
def test_token_usage(self):
result = token_usage(self.config, self.response, self.original_question)

expected_result = {
"answer": 12,
"prompt": 314,
"question": 15,
"source_documents": 1,
"total": 342
"question": 5,
"source_documents": 527,
"total": 858
}

self.assertEqual(result, expected_result)
4 changes: 4 additions & 0 deletions template.yaml
Original file line number Diff line number Diff line change
@@ -1159,3 +1159,7 @@ Resources:
AuthorizationType: NONE
RouteKey: GET /docs/v2/{proxy+}
Target: !Sub "integrations/${docsIntegration}"
Outputs:
Endpoint:
Description: "The base API endpoint for the stack"
Value: !Sub "https://${CustomDomainHost}.${CustomDomainZone}/api/v2"