From 791b358670eb2a7c5d79c87732c7bf3c86810269 Mon Sep 17 00:00:00 2001 From: jonasferoz Date: Thu, 16 May 2024 09:21:47 -0700 Subject: [PATCH 01/16] Format restricttotopic README to template --- README.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 4f69148..01336ad 100644 --- a/README.md +++ b/README.md @@ -1,17 +1,18 @@ -## Overview +# Overview | Developed by | Tryolabs | -| --- | --- | | Date of development | Feb 15, 2024 | | Validator type | Format | -| Blog | - | +| Blog | | | License | Apache 2 | | Input/Output | Output | ## Description + +### Intended Use This validator checks if a text is related with a topic. -## Requirements +### Requirements * Dependencies: - guardrails-ai>=0.4.0 @@ -25,12 +26,12 @@ This validator checks if a text is related with a topic. # Installation ```bash -guardrails hub install hub://tryolabs/restricttotopic +$ guardrails hub install hub://tryolabs/restricttotopic ``` # Usage Examples -## Validating string output via Python +### Validating string output via Python In this example, we apply the validator to a string output generated by an LLM. @@ -59,7 +60,7 @@ The Beatles were a charismatic English pop-rock band of the 1960s. """) # Validator fails ``` -## Validating JSON output via Python +### Validating JSON output via Python In this example, we apply the validator to a string field of a JSON output generated by an LLM. From 92bf1e8c35eb653249f420ee22c13c0fc99496d3 Mon Sep 17 00:00:00 2001 From: jonasferoz Date: Tue, 21 May 2024 17:59:57 -0700 Subject: [PATCH 02/16] remove unicode char, unify styling --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 01336ad..2db7f30 100644 --- a/README.md +++ b/README.md @@ -23,13 +23,13 @@ This validator checks if a text is related with a topic. * Foundation model access keys: - OPENAI_API_KEY -# Installation +## Installation ```bash $ guardrails hub install hub://tryolabs/restricttotopic ``` -# Usage Examples +## Usage Examples ### Validating string output via Python @@ -114,7 +114,7 @@ Initializes a new instance of the RestrictToTopic class.
-**`validate(self, value, metadata) → ValidationResult`** +**`validate(self, value, metadata) -> ValidationResult`** From f9fce48b04a823f7341ef4bc1de9d70e63fe8082 Mon Sep 17 00:00:00 2001 From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com> Date: Mon, 1 Jul 2024 17:02:31 -0700 Subject: [PATCH 03/16] adding inferless input_schema --- app.py | 31 +++++++++++++++++++++++++++++++ inferless.yaml | 39 +++++++++++++++++++++++++++++++++++++++ input_schema.py | 16 ++++++++++++++++ 3 files changed, 86 insertions(+) create mode 100644 app.py create mode 100644 inferless.yaml create mode 100644 input_schema.py diff --git a/app.py b/app.py new file mode 100644 index 0000000..126cbf0 --- /dev/null +++ b/app.py @@ -0,0 +1,31 @@ +import json +import torch +import nltk +from typing import Any, Dict, List +from transformers import pipeline + +class InferlessPythonModel: + + def initialize(self): + self._classifier = pipeline( + "zero-shot-classification", + model="facebook/bart-large-mnli", + device=0, + hypothesis_template="This example has to do with topic {}.", + multi_label=True, + ) + + def infer(self, inputs: Dict[str, Any]) -> Dict[str, Any]: + result = self._classifier(inputs["text"], inputs["candidate_topics"]) + topics = result["labels"] + scores = result["scores"] + found_topics = [] + for topic, score in zip(topics, scores): + if score > self._zero_shot_threshold: + found_topics.append(topic) + return {"results": found_topics} + + def finalize(self): + pass + + \ No newline at end of file diff --git a/inferless.yaml b/inferless.yaml new file mode 100644 index 0000000..59f3a40 --- /dev/null +++ b/inferless.yaml @@ -0,0 +1,39 @@ +# Inferless config file (version: 1.0.0) +version: 1.0.0 + +name: Restrict-To-Topic +import_source: GIT + +# you can choose the options between ONNX, TENSORFLOW, PYTORCH +source_framework_type: PYTORCH + +configuration: + # if you want to use a custom runtime, add the runtime id below. + # you can find it by running `inferless r6897f8untime list` or create one with `inferless runtime upload` and update this file it by running `inferless runtime select --id `. + custom_runtime_id: 035c210b-3425-43d7-be4f-f341fae13842 + custom_runtime_version: '0' + + # if you want to use a custom volume, add the volume id and name below, + # you can find it by running `inferless volume list` or create one with `inferless volume create -n {VOLUME_NAME}` + custom_volume_id: '' + custom_volume_name: '' + + gpu_type: T4 + inference_time: '180' + is_dedicated: false + is_serverless: false + max_replica: '1' + min_replica: '0' + scale_down_delay: '600' + region: region-1 + vcpu: '1.5' + ram: '7' +env: + # Add your environment variables here + # ENV: 'PROD' +secrets: + # Add your secret ids here you can find it by running `inferless secrets list` + # - 65723205-ce21-4392-a10b-3tf00c58988c +io_schema: true +model_url: https://github.com/guardrails-ai/restricttotopic +provider: GITHUB diff --git a/input_schema.py b/input_schema.py new file mode 100644 index 0000000..de156b9 --- /dev/null +++ b/input_schema.py @@ -0,0 +1,16 @@ +INPUT_SCHEMA = { + "text": { + "example": [ + "Let's talk about Iphones made by Apple" + ], + "shape": [1], + "datatype": "STRING", + "required": True, + }, + "candidate_topics": { + "example": ["Apple Iphone", "Samsung Galaxy"], + "shape": [-1], + "datatype": "STRING", + "required": True, + }, +} From 7d19707cf1385af0a8e099a682840aeac51606e4 Mon Sep 17 00:00:00 2001 From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com> Date: Mon, 1 Jul 2024 17:09:49 -0700 Subject: [PATCH 04/16] Adding threshold --- app.py | 3 ++- input_schema.py | 10 +++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/app.py b/app.py index 126cbf0..6138095 100644 --- a/app.py +++ b/app.py @@ -19,9 +19,10 @@ def infer(self, inputs: Dict[str, Any]) -> Dict[str, Any]: result = self._classifier(inputs["text"], inputs["candidate_topics"]) topics = result["labels"] scores = result["scores"] + zero_shot_threshold = result["zero_shot_threshold"] found_topics = [] for topic, score in zip(topics, scores): - if score > self._zero_shot_threshold: + if score > zero_shot_threshold: found_topics.append(topic) return {"results": found_topics} diff --git a/input_schema.py b/input_schema.py index de156b9..013d75c 100644 --- a/input_schema.py +++ b/input_schema.py @@ -1,8 +1,6 @@ INPUT_SCHEMA = { "text": { - "example": [ - "Let's talk about Iphones made by Apple" - ], + "example": ["Let's talk about Iphones made by Apple"], "shape": [1], "datatype": "STRING", "required": True, @@ -13,4 +11,10 @@ "datatype": "STRING", "required": True, }, + "zero_shot_threshold": { + "example": [0.5], + "shape": [1], + "datatype": "FP32", + "required": True, + }, } From 11afb48688151f1c0c5978d669989ed8fb112dd4 Mon Sep 17 00:00:00 2001 From: Wyatt Lansford <22553069+wylansford@users.noreply.github.com> Date: Mon, 1 Jul 2024 17:13:35 -0700 Subject: [PATCH 05/16] fixing where to pull threshold key --- app.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app.py b/app.py index 6138095..40952ac 100644 --- a/app.py +++ b/app.py @@ -19,10 +19,9 @@ def infer(self, inputs: Dict[str, Any]) -> Dict[str, Any]: result = self._classifier(inputs["text"], inputs["candidate_topics"]) topics = result["labels"] scores = result["scores"] - zero_shot_threshold = result["zero_shot_threshold"] found_topics = [] for topic, score in zip(topics, scores): - if score > zero_shot_threshold: + if score > inputs["zero_shot_threshold"]: found_topics.append(topic) return {"results": found_topics} From f8c43ca60ef409abb96ee9aeafec7d73490bc129 Mon Sep 17 00:00:00 2001 From: Aarav Navani <38411399+oofmeister27@users.noreply.github.com> Date: Fri, 19 Jul 2024 15:56:49 -0700 Subject: [PATCH 06/16] add error spans --- validator/main.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/validator/main.py b/validator/main.py index 8a796cc..4809290 100644 --- a/validator/main.py +++ b/validator/main.py @@ -5,6 +5,7 @@ from dotenv import load_dotenv from guardrails.validator_base import ( + ErrorSpan, FailResult, PassResult, ValidationResult, @@ -341,12 +342,28 @@ def validate( elif topic in self._invalid_topics: invalid_topics_found.append(topic) + error_spans = [] + # Require at least one valid topic and no invalid topics if invalid_topics_found: + error_spans.append( + ErrorSpan( + start=value.find(topic), + end=value.find(topic) + len(topic), + reason=f"Text contains invalid topic: {topic}", + ) + ) return FailResult( - error_message=f"Invalid topics found: {invalid_topics_found}" + error_message=f"Invalid topics found: {invalid_topics_found}", + error_spans=error_spans ) if not valid_topics_found: - return FailResult(error_message="No valid topic was found.") - + return FailResult( + error_message="No valid topic was found.", + error_spans=[ErrorSpan( + start=0, + end=len(value), + reason="No valid topic was found." + )] + ) return PassResult() From 730762cb2cce9af23d00a70e291bd1d68d6d98e2 Mon Sep 17 00:00:00 2001 From: Aarav Navani <38411399+oofmeister27@users.noreply.github.com> Date: Fri, 19 Jul 2024 16:17:41 -0700 Subject: [PATCH 07/16] loop --- validator/main.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/validator/main.py b/validator/main.py index 4809290..60288d6 100644 --- a/validator/main.py +++ b/validator/main.py @@ -346,13 +346,14 @@ def validate( # Require at least one valid topic and no invalid topics if invalid_topics_found: - error_spans.append( - ErrorSpan( - start=value.find(topic), - end=value.find(topic) + len(topic), - reason=f"Text contains invalid topic: {topic}", + for topic in invalid_topics_found: + error_spans.append( + ErrorSpan( + start=value.find(topic), + end=value.find(topic) + len(topic), + reason=f"Text contains invalid topic: {topic}", + ) ) - ) return FailResult( error_message=f"Invalid topics found: {invalid_topics_found}", error_spans=error_spans From 935df9baac7f767a2284acf3a30e834432d8e747 Mon Sep 17 00:00:00 2001 From: Aarav Navani <38411399+oofmeister27@users.noreply.github.com> Date: Thu, 1 Aug 2024 23:06:28 -0700 Subject: [PATCH 08/16] change to app.py setup --- app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app.py b/app.py index 40952ac..6a2eb9c 100644 --- a/app.py +++ b/app.py @@ -23,6 +23,8 @@ def infer(self, inputs: Dict[str, Any]) -> Dict[str, Any]: for topic, score in zip(topics, scores): if score > inputs["zero_shot_threshold"]: found_topics.append(topic) + if not found_topics: + return {"results": ["No valid topic found."]} return {"results": found_topics} def finalize(self): From 6b7b3e7b80fd1bfdb29267e3c3296cc06622fc05 Mon Sep 17 00:00:00 2001 From: Aarav Navani <38411399+aaravnavani@users.noreply.github.com> Date: Thu, 8 Aug 2024 10:56:19 -0700 Subject: [PATCH 09/16] add cuda --- app.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app.py b/app.py index 6a2eb9c..e10197e 100644 --- a/app.py +++ b/app.py @@ -14,6 +14,7 @@ def initialize(self): hypothesis_template="This example has to do with topic {}.", multi_label=True, ) + self._classifier.to("cuda") def infer(self, inputs: Dict[str, Any]) -> Dict[str, Any]: result = self._classifier(inputs["text"], inputs["candidate_topics"]) @@ -30,4 +31,4 @@ def infer(self, inputs: Dict[str, Any]) -> Dict[str, Any]: def finalize(self): pass - \ No newline at end of file + From d5cdfabeb5dfac8f6e64d8d8cb34a341ac939c13 Mon Sep 17 00:00:00 2001 From: Aarav Navani <38411399+aaravnavani@users.noreply.github.com> Date: Thu, 8 Aug 2024 11:17:53 -0700 Subject: [PATCH 10/16] change cuda implementation --- app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app.py b/app.py index e10197e..4f7914a 100644 --- a/app.py +++ b/app.py @@ -10,11 +10,11 @@ def initialize(self): self._classifier = pipeline( "zero-shot-classification", model="facebook/bart-large-mnli", - device=0, + device="cuda", hypothesis_template="This example has to do with topic {}.", multi_label=True, ) - self._classifier.to("cuda") + #self._classifier.to("cuda") def infer(self, inputs: Dict[str, Any]) -> Dict[str, Any]: result = self._classifier(inputs["text"], inputs["candidate_topics"]) From 59e9d47cb71291d025087d888b011a7b72d68722 Mon Sep 17 00:00:00 2001 From: Aarav Navani Date: Wed, 14 Aug 2024 12:56:13 -0700 Subject: [PATCH 11/16] fastapi setup --- app.py | 102 +++++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 73 insertions(+), 29 deletions(-) diff --git a/app.py b/app.py index 4f7914a..aab6985 100644 --- a/app.py +++ b/app.py @@ -1,34 +1,78 @@ -import json -import torch -import nltk -from typing import Any, Dict, List + + +from fastapi import FastAPI, HTTPException +from pydantic import BaseModel +from typing import List, Union from transformers import pipeline +import torch -class InferlessPythonModel: +app = FastAPI() - def initialize(self): - self._classifier = pipeline( - "zero-shot-classification", - model="facebook/bart-large-mnli", - device="cuda", - hypothesis_template="This example has to do with topic {}.", - multi_label=True, - ) - #self._classifier.to("cuda") - - def infer(self, inputs: Dict[str, Any]) -> Dict[str, Any]: - result = self._classifier(inputs["text"], inputs["candidate_topics"]) - topics = result["labels"] - scores = result["scores"] - found_topics = [] - for topic, score in zip(topics, scores): - if score > inputs["zero_shot_threshold"]: - found_topics.append(topic) - if not found_topics: - return {"results": ["No valid topic found."]} - return {"results": found_topics} - - def finalize(self): - pass +# Initialize the zero-shot classification pipeline +classifier = pipeline( + "zero-shot-classification", + model="facebook/bart-large-mnli", + device=torch.device("cuda" if torch.cuda.is_available() else "cpu"), + hypothesis_template="This example has to do with topic {}.", + multi_label=True, +) + +class InferenceData(BaseModel): + name: str + shape: List[int] + data: Union[List[str], List[float]] + datatype: str + +class InputRequest(BaseModel): + inputs: List[InferenceData] + +class OutputResponse(BaseModel): + modelname: str + modelversion: str + outputs: List[InferenceData] + +@app.post("/validate", response_model=OutputResponse) +async def restrict_to_topic(input_request: InputRequest): + print('make request') + text = None + candidate_topics = None + zero_shot_threshold = 0.5 + + for inp in input_request.inputs: + if inp.name == "text": + text = inp.data[0] + elif inp.name == "candidate_topics": + candidate_topics = inp.data + elif inp.name == "zero_shot_threshold": + zero_shot_threshold = float(inp.data[0]) + + if text is None or candidate_topics is None: + raise HTTPException(status_code=400, detail="Invalid input format") + + # Perform zero-shot classification + result = classifier(text, candidate_topics) + topics = result["labels"] + scores = result["scores"] + found_topics = [topic for topic, score in zip(topics, scores) if score > zero_shot_threshold] + if not found_topics: + found_topics = ["No valid topic found."] + output_data = OutputResponse( + modelname="RestrictToTopicModel", + modelversion="1", + outputs=[ + InferenceData( + name="results", + datatype="BYTES", + shape=[len(found_topics)], + data=found_topics + ) + ] + ) + + print(f"Output data: {output_data}") + return output_data + +# Run the app with uvicorn +# Save this script as app.py and run with: uvicorn app:app --reload \ No newline at end of file From 10298c7f0f55800795776b97b3fc8b11a08fe56b Mon Sep 17 00:00:00 2001 From: Aarav Navani Date: Wed, 14 Aug 2024 12:57:55 -0700 Subject: [PATCH 12/16] newline --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index aab6985..5f39f91 100644 --- a/app.py +++ b/app.py @@ -75,4 +75,4 @@ async def restrict_to_topic(input_request: InputRequest): return output_data # Run the app with uvicorn -# Save this script as app.py and run with: uvicorn app:app --reload \ No newline at end of file +# Save this script as app.py and run with: uvicorn app:app --reload From a0680f82599e19f53eb4d2ebbdda594878e606f4 Mon Sep 17 00:00:00 2001 From: Aarav Navani Date: Thu, 22 Aug 2024 10:45:06 -0700 Subject: [PATCH 13/16] update app.py --- app.py | 8 +++----- test.py | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 test.py diff --git a/app.py b/app.py index 5f39f91..a8f818a 100644 --- a/app.py +++ b/app.py @@ -1,5 +1,3 @@ - - from fastapi import FastAPI, HTTPException from pydantic import BaseModel from typing import List, Union @@ -32,7 +30,7 @@ class OutputResponse(BaseModel): outputs: List[InferenceData] @app.post("/validate", response_model=OutputResponse) -async def restrict_to_topic(input_request: InputRequest): +def restrict_to_topic(input_request: InputRequest): print('make request') text = None candidate_topics = None @@ -51,6 +49,7 @@ async def restrict_to_topic(input_request: InputRequest): # Perform zero-shot classification result = classifier(text, candidate_topics) + print("result: ", result) topics = result["labels"] scores = result["scores"] found_topics = [topic for topic, score in zip(topics, scores) if score > zero_shot_threshold] @@ -71,8 +70,7 @@ async def restrict_to_topic(input_request: InputRequest): ] ) - print(f"Output data: {output_data}") return output_data # Run the app with uvicorn -# Save this script as app.py and run with: uvicorn app:app --reload +# Save this script as app.py and run with: uvicorn app:app --reload \ No newline at end of file diff --git a/test.py b/test.py new file mode 100644 index 0000000..3544699 --- /dev/null +++ b/test.py @@ -0,0 +1,16 @@ +# Import Guard and Validator +from guardrails.hub import RelevancyEvaluator +from guardrails import Guard + +# Setup Guard +guard = Guard().use( + RelevancyEvaluator(llm_callable="gpt-3.5-turbo") +) + +# Example values +value = { + "original_prompt": "What is the capital of France?", + "reference_text": "The capital of France is Paris." +} + +guard.validate(metadata=value) # Validator passes if the text is relevant \ No newline at end of file From 2e941c8e47fb6859ac7ab84cd27dfa47e05c621b Mon Sep 17 00:00:00 2001 From: Aarav Navani Date: Thu, 22 Aug 2024 10:49:38 -0700 Subject: [PATCH 14/16] remove test --- test.py | 16 ---------------- 1 file changed, 16 deletions(-) delete mode 100644 test.py diff --git a/test.py b/test.py deleted file mode 100644 index 3544699..0000000 --- a/test.py +++ /dev/null @@ -1,16 +0,0 @@ -# Import Guard and Validator -from guardrails.hub import RelevancyEvaluator -from guardrails import Guard - -# Setup Guard -guard = Guard().use( - RelevancyEvaluator(llm_callable="gpt-3.5-turbo") -) - -# Example values -value = { - "original_prompt": "What is the capital of France?", - "reference_text": "The capital of France is Paris." -} - -guard.validate(metadata=value) # Validator passes if the text is relevant \ No newline at end of file From 86ef8d3f3807128bbd3f89a132b8d435f1609d71 Mon Sep 17 00:00:00 2001 From: Aarav Navani Date: Thu, 22 Aug 2024 10:49:56 -0700 Subject: [PATCH 15/16] newline --- app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.py b/app.py index a8f818a..5f3325c 100644 --- a/app.py +++ b/app.py @@ -73,4 +73,4 @@ def restrict_to_topic(input_request: InputRequest): return output_data # Run the app with uvicorn -# Save this script as app.py and run with: uvicorn app:app --reload \ No newline at end of file +# Save this script as app.py and run with: uvicorn app:app --reload From feb730cdd0c852396813df9bc404dc54845347cb Mon Sep 17 00:00:00 2001 From: Aarav Navani Date: Mon, 26 Aug 2024 13:24:32 -0700 Subject: [PATCH 16/16] address comments --- app.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/app.py b/app.py index 5f3325c..0d94b82 100644 --- a/app.py +++ b/app.py @@ -3,14 +3,18 @@ from typing import List, Union from transformers import pipeline import torch +import os app = FastAPI() +env = os.environ.get("env", "dev") +torch_device = "cuda" if env == "prod" else "cpu" + # Initialize the zero-shot classification pipeline classifier = pipeline( "zero-shot-classification", model="facebook/bart-large-mnli", - device=torch.device("cuda" if torch.cuda.is_available() else "cpu"), + device=torch.device(torch_device), hypothesis_template="This example has to do with topic {}.", multi_label=True, ) @@ -22,7 +26,9 @@ class InferenceData(BaseModel): datatype: str class InputRequest(BaseModel): - inputs: List[InferenceData] + text: str + candidate_topics: List[str] + zero_shot_threshold: float = 0.5 class OutputResponse(BaseModel): modelname: str @@ -32,17 +38,11 @@ class OutputResponse(BaseModel): @app.post("/validate", response_model=OutputResponse) def restrict_to_topic(input_request: InputRequest): print('make request') - text = None - candidate_topics = None - zero_shot_threshold = 0.5 - for inp in input_request.inputs: - if inp.name == "text": - text = inp.data[0] - elif inp.name == "candidate_topics": - candidate_topics = inp.data - elif inp.name == "zero_shot_threshold": - zero_shot_threshold = float(inp.data[0]) + text = input_request.text + candidate_topics = input_request.candidate_topics + zero_shot_threshold = input_request.zero_shot_threshold + if text is None or candidate_topics is None: raise HTTPException(status_code=400, detail="Invalid input format")