From c43babe1cdec3d7f6f31ee817b989f074472fb59 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Wed, 13 Dec 2023 12:37:36 -0600 Subject: [PATCH 01/20] start exception handling --- guardrails/classes/history/call.py | 18 ++- guardrails/classes/validation_outcome.py | 13 +- guardrails/guard.py | 18 +-- guardrails/run.py | 35 +++-- guardrails/validators/on_topic.py | 4 +- guardrails/validators/pii_filter.py | 4 +- .../integration_tests/test_data_validation.py | 79 ++++++---- tests/integration_tests/test_datatypes.py | 21 +-- .../test_embedding_openai.py | 7 +- tests/integration_tests/test_guard.py | 11 +- tests/integration_tests/test_validators.py | 77 ++++++---- .../validators/test_on_topic.py | 2 + tests/unit_tests/test_guard.py | 25 ++- tests/unit_tests/test_validators.py | 144 +++++++++++++----- 14 files changed, 304 insertions(+), 154 deletions(-) diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py index b60d337cf..07645adfc 100644 --- a/guardrails/classes/history/call.py +++ b/guardrails/classes/history/call.py @@ -1,4 +1,5 @@ from typing import Dict, Optional, Union +from pydantic import PrivateAttr from pydantic import Field from rich.panel import Panel @@ -29,6 +30,7 @@ class Call(ArbitraryModel): inputs: CallInputs = Field( description="The inputs as passed in to Guard.__call__ or Guard.parse" ) + _exception: Exception = PrivateAttr() # Prevent Pydantic from changing our types # Without this, Pydantic casts iterations to a list @@ -39,9 +41,10 @@ def __init__( ): iterations = iterations or Stack() inputs = inputs or CallInputs() - super().__init__(iterations=iterations, inputs=inputs) # type: ignore + super().__init__(iterations=iterations, inputs=inputs, _exception=None) # type: ignore self.iterations = iterations self.inputs = inputs + self._exception = None @property def prompt(self) -> Optional[str]: @@ -277,17 +280,26 @@ def validator_logs(self) -> Stack[ValidatorLogs]: def error(self) -> Optional[str]: """The error message from any exception that raised and interrupted the run.""" - if self.iterations.empty(): + if self._exception: + return str(self._exception) + elif self.iterations.empty(): return None return self.iterations.last.error # type: ignore + @property def exception(self) -> Optional[Exception]: """The exception that interrupted the run.""" - if self.iterations.empty(): + if self._exception: + return self._exception + elif self.iterations.empty(): return None return self.iterations.last.exception # type: ignore + @exception.setter + def exception(self, value: Exception) -> None: + self._exception = value + @property def failed_validations(self) -> Stack[ValidatorLogs]: """The validator logs for any validations that failed during the diff --git a/guardrails/classes/validation_outcome.py b/guardrails/classes/validation_outcome.py index 538280be8..540b90096 100644 --- a/guardrails/classes/validation_outcome.py +++ b/guardrails/classes/validation_outcome.py @@ -2,7 +2,7 @@ from pydantic import Field -from guardrails.classes.history import Call +from guardrails.classes.history import Call, Iteration from guardrails.classes.output_type import OT from guardrails.constants import pass_status from guardrails.utils.logs_utils import ArbitraryModel @@ -32,15 +32,12 @@ class ValidationOutcome(Generic[OT], ArbitraryModel): error: Optional[str] = Field(default=None) @classmethod - def from_guard_history(cls, call: Call, error_message: Optional[str]): - last_output = ( - call.iterations.last.validation_output - if not call.iterations.empty() and call.iterations.last is not None - else None - ) + def from_guard_history(cls, call: Call): + last_iteration = call.iterations.last or Iteration() + last_output = last_iteration.validation_output or last_iteration.parsed_output validation_passed = call.status == pass_status reask = last_output if isinstance(last_output, ReAsk) else None - error = call.error or error_message + error = call.error output = cast(OT, call.validated_output) return cls( raw_llm_output=call.raw_outputs.last, diff --git a/guardrails/guard.py b/guardrails/guard.py index 658758ce0..ab6def4c2 100644 --- a/guardrails/guard.py +++ b/guardrails/guard.py @@ -422,10 +422,8 @@ def _call_sync( base_model=self.base_model, full_schema_reask=full_schema_reask, ) - call, error_message = runner( - call_log=call_log, prompt_params=prompt_params - ) - return ValidationOutcome[OT].from_guard_history(call, error_message) + call = runner(call_log=call_log, prompt_params=prompt_params) + return ValidationOutcome[OT].from_guard_history(call) async def _call_async( self, @@ -483,10 +481,10 @@ async def _call_async( base_model=self.base_model, full_schema_reask=full_schema_reask, ) - call, error_message = await runner.async_run( + call = await runner.async_run( call_log=call_log, prompt_params=prompt_params ) - return ValidationOutcome[OT].from_guard_history(call, error_message) + return ValidationOutcome[OT].from_guard_history(call) def __repr__(self): return f"Guard(RAIL={self.rail})" @@ -662,9 +660,9 @@ def _sync_parse( base_model=self.base_model, full_schema_reask=full_schema_reask, ) - call, error_message = runner(call_log=call_log, prompt_params=prompt_params) + call = runner(call_log=call_log, prompt_params=prompt_params) - return ValidationOutcome[OT].from_guard_history(call, error_message) + return ValidationOutcome[OT].from_guard_history(call) async def _async_parse( self, @@ -704,11 +702,11 @@ async def _async_parse( base_model=self.base_model, full_schema_reask=full_schema_reask, ) - call, error_message = await runner.async_run( + call = await runner.async_run( call_log=call_log, prompt_params=prompt_params ) - return ValidationOutcome[OT].from_guard_history(call, error_message) + return ValidationOutcome[OT].from_guard_history(call) def with_prompt_validation( self, diff --git a/guardrails/run.py b/guardrails/run.py index fc3ac77b7..b02f4e884 100644 --- a/guardrails/run.py +++ b/guardrails/run.py @@ -102,9 +102,7 @@ def __init__( self.base_model = base_model self.full_schema_reask = full_schema_reask - def __call__( - self, call_log: Call, prompt_params: Optional[Dict] = None - ) -> Tuple[Call, Optional[str]]: + def __call__(self, call_log: Call, prompt_params: Optional[Dict] = None) -> Call: """Execute the runner by repeatedly calling step until the reask budget is exhausted. @@ -115,7 +113,6 @@ def __call__( Returns: The Call log for this run. """ - error_message = None try: if prompt_params is None: prompt_params = {} @@ -198,10 +195,16 @@ def __call__( include_instructions=include_instructions, ) except UserFacingException as e: + call_log.exception = e.original_exception + # import traceback + # traceback.print_exception(e) raise e.original_exception except Exception as e: - error_message = str(e) - return call_log, error_message + call_log.exception = e + # import traceback + # traceback.print_exception(e) + raise e + return call_log def step( self, @@ -311,6 +314,8 @@ def step( error_message = str(e) iteration.outputs.error = error_message iteration.outputs.exception = e + # import traceback + # traceback.print_exception(e) raise e return iteration @@ -684,7 +689,7 @@ def __init__( async def async_run( self, call_log: Call, prompt_params: Optional[Dict] = None - ) -> Tuple[Call, Optional[str]]: + ) -> Call: """Execute the runner by repeatedly calling step until the reask budget is exhausted. @@ -695,7 +700,6 @@ async def async_run( Returns: The Call log for this run. """ - error_message = None try: if prompt_params is None: prompt_params = {} @@ -773,11 +777,17 @@ async def async_run( prompt_params=prompt_params, ) except UserFacingException as e: + call_log.exception = e.original_exception + # import traceback + # traceback.print_exception(e) raise e.original_exception except Exception as e: - error_message = str(e) + call_log.exception = e + # import traceback + # traceback.print_exception(e) + raise e - return call_log, error_message + return call_log async def async_step( self, @@ -857,6 +867,9 @@ async def async_step( # Parse: parse the output. parsed_output, parsing_error = self.parse(index, output, output_schema) if parsing_error: + # Parsing errors are captured and not raised + # because they are recoverable + # i.e. result in a reask iteration.outputs.exception = parsing_error iteration.outputs.error = str(parsing_error) @@ -883,6 +896,8 @@ async def async_step( error_message = str(e) iteration.outputs.error = error_message iteration.outputs.exception = e + # import traceback + # traceback.print_exception(e) raise e return iteration diff --git a/guardrails/validators/on_topic.py b/guardrails/validators/on_topic.py index cef227cc9..eeeb54d17 100644 --- a/guardrails/validators/on_topic.py +++ b/guardrails/validators/on_topic.py @@ -143,7 +143,7 @@ def set_client(self): # Set the OpenAI API base if specified if api_base: - openai.api_version = api_base + openai.api_base = api_base # todo: extract some of these similar methods into a base class w provenance @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(5)) @@ -242,6 +242,8 @@ def validate(self, value: str, metadata: Dict[str, Any]) -> ValidationResult: candidate_topics = valid_topics.union(invalid_topics) # Check which model(s) to use + print("self._disable_classifier: ", self._disable_classifier) + print("self._disable_llm: ", self._disable_llm) if self._disable_classifier and self._disable_llm: # Error, no model set raise ValueError("Either classifier or llm must be enabled.") elif ( diff --git a/guardrails/validators/pii_filter.py b/guardrails/validators/pii_filter.py index 014355c49..38cd2a34b 100644 --- a/guardrails/validators/pii_filter.py +++ b/guardrails/validators/pii_filter.py @@ -114,13 +114,13 @@ def validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult: entities_to_filter = self.PII_ENTITIES_MAP.get(pii_entities, None) if entities_to_filter is None: raise ValueError( - f"`pii_entities` must be one of {self.PII_ENTITIES_MAP.keys()}" + f"`pii_entities` must be one of {list(self.PII_ENTITIES_MAP.keys())}" ) elif isinstance(pii_entities, list): entities_to_filter = pii_entities else: raise ValueError( - f"`pii_entities` must be one of {self.PII_ENTITIES_MAP.keys()}" + f"`pii_entities` must be one of {list(self.PII_ENTITIES_MAP.keys())}" " or a list of strings." ) diff --git a/tests/integration_tests/test_data_validation.py b/tests/integration_tests/test_data_validation.py index 92d8e6bb8..226ac8f1d 100644 --- a/tests/integration_tests/test_data_validation.py +++ b/tests/integration_tests/test_data_validation.py @@ -6,30 +6,33 @@ from guardrails import Guard from guardrails.utils.reask_utils import ReAsk +from guardrails.validator_base import ValidatorError from guardrails.validators import ValidChoices test_cases = [ - ('{"choice": {"action": "fight", "fight_move": "kick"}}', False, False), + ('{"choice": {"action": "fight", "fight_move": "kick"}}', False, False, False), ( '{"choice": {"action": "flight", "flight_direction": "north", "flight_speed": 1}}', False, False, + False, ), - ('{"choice": {"action": "flight", "fight_move": "punch"}}', False, True), + ('{"choice": {"action": "flight", "fight_move": "punch"}}', False, True, False), ( '{"choice": {"action": "fight", "flight_direction": "north", "flight_speed": 1}}', False, True, + False, ), - ('{"choice": {"action": "random_action"}}', False, True), - ('{"choice": {"action": "fight", "fight_move": "random_move"}}', True, True), - ('{"choice": {"action": "flight", "random_key": "random_value"}}', False, True), - ('{"choice": {"action": "flight", "random_key": "random_value"}', True, True), + ('{"choice": {"action": "random_action"}}', False, True, False), + ('{"choice": {"action": "fight", "fight_move": "random_move"}}', True, True, True), + ('{"choice": {"action": "flight", "random_key": "random_value"}}', False, True, False), + ('{"choice": {"action": "flight", "random_key": "random_value"}', False, True, True), ] -@pytest.mark.parametrize("llm_output, raises, fails", test_cases) -def test_choice_validation(llm_output, raises, fails): +@pytest.mark.parametrize("llm_output, raises, fails, has_error", test_cases) +def test_choice_validation(llm_output, raises, fails, has_error): rail_spec = """ @@ -55,22 +58,28 @@ def test_choice_validation(llm_output, raises, fails): guard = Guard.from_rail_string(rail_spec) # If raises is True, then the test should raise an exception. - result = guard.parse(llm_output, num_reasks=0) + # For our existing test cases this will always be a ValidatorError if raises: - assert result.validation_passed is False - assert result.error is not None - elif fails: - assert result.validation_passed is False - assert result.reask is not None - assert result.error is None + with pytest.raises(ValidatorError): + guard.parse(llm_output, num_reasks=0) else: - assert result.validation_passed is True - assert result.validated_output is not None - assert not isinstance(result.validated_output, ReAsk) - - -@pytest.mark.parametrize("llm_output, raises, fails", test_cases) -def test_choice_validation_pydantic(llm_output, raises, fails): + result = guard.parse(llm_output, num_reasks=0) + if fails and not has_error: + assert result.validation_passed is False + assert result.reask is not None + assert result.error is None + elif fails and has_error: + assert result.validation_passed is False + assert result.reask is not None + assert result.error is not None + else: + assert result.validation_passed is True + assert result.validated_output is not None + assert not isinstance(result.validated_output, ReAsk) + + +@pytest.mark.parametrize("llm_output, raises, fails, has_error", test_cases) +def test_choice_validation_pydantic(llm_output, raises, has_error, fails): class Fight(BaseModel): action: Literal["fight"] fight_move: str = Field( @@ -96,15 +105,21 @@ class Choice(BaseModel): guard = Guard.from_pydantic(output_class=Choice, prompt="Dummy prompt.") # If raises is True, then the test should raise an exception. - result = guard.parse(llm_output, num_reasks=0) + # For our existing test cases this will always be a ValidatorError if raises: - assert result.validation_passed is False - assert result.error is not None - elif fails: - assert result.validation_passed is False - assert result.reask is not None - assert result.error is None + with pytest.raises(ValidatorError): + guard.parse(llm_output, num_reasks=0) else: - assert result.validation_passed is True - assert result.validated_output is not None - assert not isinstance(result.validated_output, ReAsk) + result = guard.parse(llm_output, num_reasks=0) + if fails and not has_error: + assert result.validation_passed is False + assert result.reask is not None + assert result.error is None + elif fails and has_error: + assert result.validation_passed is False + assert result.reask is not None + assert result.error is not None + else: + assert result.validation_passed is True + assert result.validated_output is not None + assert not isinstance(result.validated_output, ReAsk) diff --git a/tests/integration_tests/test_datatypes.py b/tests/integration_tests/test_datatypes.py index bb830b527..85bb1f099 100644 --- a/tests/integration_tests/test_datatypes.py +++ b/tests/integration_tests/test_datatypes.py @@ -1,5 +1,6 @@ import pytest +from dateutil.parser import ParserError from guardrails.guard import Guard @@ -57,13 +58,13 @@ def test_defaulted_date_parser(date_string: str): @pytest.mark.parametrize( - "date_string", + "date_string,error_type", [ - ("1696343743"), # Unix timestamp/seconds - ("1697579939213"), # Unix timestamp/milliseconds + ("1696343743", ParserError), # Unix timestamp/seconds + ("1697579939213", OverflowError), # Unix timestamp/milliseconds ], ) -def test_defaulted_date_parser_unsupported_values(date_string: str): +def test_defaulted_date_parser_unsupported_values(date_string: str, error_type: Exception): rail_spec = """ @@ -80,9 +81,9 @@ def test_defaulted_date_parser_unsupported_values(date_string: str): """ guard = Guard.from_rail_string(rail_spec) - response = guard.parse( - llm_output='{"name": "John Doe", "dob": "' + date_string + '"}', - num_reasks=0, - ) - assert response.error is not None - # this should always raise either a ValueError or an OverflowError + with pytest.raises(Exception) as excinfo: + guard.parse( + llm_output='{"name": "John Doe", "dob": "' + date_string + '"}', + num_reasks=0, + ) + assert isinstance(excinfo.value, error_type) is True diff --git a/tests/integration_tests/test_embedding_openai.py b/tests/integration_tests/test_embedding_openai.py index 0fffd8922..8fad453d6 100644 --- a/tests/integration_tests/test_embedding_openai.py +++ b/tests/integration_tests/test_embedding_openai.py @@ -30,6 +30,10 @@ def __init__(self, data=None): def json(self): return {"data": self.data} + + def __getitem__(self, key: str): + return getattr(self, key) + @pytest.mark.skipif( @@ -69,10 +73,11 @@ def test__get_embedding(self, mocker): mock_create = None if OPENAI_VERSION.startswith("0"): mock_create = mocker.patch("openai.Embedding.create") + mock_create.return_value = MockResponse(data=[{"embedding": [1.0, 2.0, 3.0]}]) else: mock_create = mocker.patch("openai.resources.Embeddings.create") + mock_create.return_value = MockResponse(data=[[1.0, 2.0, 3.0]]) - mock_create.return_value = MockResponse(data=[[1.0, 2.0, 3.0]]) instance = OpenAIEmbedding(api_key="test_api_key") result = instance._get_embedding(["test text"]) diff --git a/tests/integration_tests/test_guard.py b/tests/integration_tests/test_guard.py index 91eac63a1..9a19b4861 100644 --- a/tests/integration_tests/test_guard.py +++ b/tests/integration_tests/test_guard.py @@ -781,9 +781,10 @@ class Task(BaseModel): assert dict_o == {"status": "not started"} guard = gd.Guard.from_pydantic(Task) - response = guard( - get_static_openai_create_func(), - prompt="What is the status of this task REALLY?", - ) + with pytest.raises(ValueError) as excinfo: + guard( + get_static_openai_create_func(), + prompt="What is the status of this task REALLY?", + ) - assert response.error is not None + assert str(excinfo.value).startswith("Invalid enum value") is True diff --git a/tests/integration_tests/test_validators.py b/tests/integration_tests/test_validators.py index 111314013..a1907b91c 100644 --- a/tests/integration_tests/test_validators.py +++ b/tests/integration_tests/test_validators.py @@ -82,19 +82,24 @@ def embed_function(text: str): # 1.2 Test not passing prev_values # Should raise ValueError val = "3" - output = guard.parse( - llm_output=val, - ) - assert output.error is not None + with pytest.raises(ValueError) as excinfo: + guard.parse( + llm_output=val, + ) + assert str(excinfo.value) == "You must provide a list of previous values in metadata." # 1.3 Test passing str prev values for int val # Should raise ValueError val = "3" - output = guard.parse( - llm_output=val, - metadata={"prev_values": [str(i) for i in int_prev_values]}, + with pytest.raises(ValueError) as excinfo: + guard.parse( + llm_output=val, + metadata={"prev_values": [str(i) for i in int_prev_values]}, + ) + assert str(excinfo.value) == ( + "Both given value and all the previous values must be " + "integers in order to use the distribution check validator." ) - assert output.error is not None # 1.4 Test for values outside the standard deviation val = "300" @@ -116,29 +121,38 @@ def embed_function(text: str): # 2.2 Test not passing prev_values # Should raise ValueError val = "cisco" - output = guard.parse( - llm_output=val, - metadata={"embed_function": embed_function}, - ) - assert output.error is not None + with pytest.raises(ValueError) as excinfo: + guard.parse( + llm_output=val, + metadata={"embed_function": embed_function}, + ) + assert str(excinfo.value) == "You must provide a list of previous values in metadata." # 2.3 Test passing int prev values for str val # Should raise ValueError val = "cisco" - output = guard.parse( - llm_output=val, - metadata={"prev_values": int_prev_values, "embed_function": embed_function}, + with pytest.raises(ValueError) as excinfo: + guard.parse( + llm_output=val, + metadata={"prev_values": int_prev_values, "embed_function": embed_function}, + ) + assert str(excinfo.value) == ( + "Both given value and all the previous values must be " + "strings in order to use the distribution check validator." ) - assert output.error is not None # 2.4 Test not pasisng embed_function # Should raise ValueError val = "cisco" - output = guard.parse( - llm_output=val, - metadata={"prev_values": str_prev_values}, + with pytest.raises(ValueError) as excinfo: + guard.parse( + llm_output=val, + metadata={"prev_values": str_prev_values}, + ) + assert str(excinfo.value) == ( + "You must provide `embed_function` in metadata in order to " + "check the semantic similarity of the generated string." ) - assert output.error is not None # 2.5 Test for values outside the standard deviation val = "taj mahal" @@ -285,10 +299,16 @@ def test_pii_filter(mocker): ) text = "My email address is demo@lol.com, and my phone number is 1234567890" - output = guard.parse( - llm_output=text, + with pytest.raises(ValueError) as excinfo: + guard.parse( + llm_output=text, + ) + assert str(excinfo.value) == ( + "`pii_entities` must be set in order to use the `PIIFilter` validator." + "Add this: `pii_entities=['PERSON', 'PHONE_NUMBER']`" + "OR pii_entities='pii' or 'spi'" + "in init or metadata." ) - assert output.error is not None # ------------------ # 4. Initialise Guard from string without setting pii_entities @@ -360,10 +380,11 @@ def test_pii_filter(mocker): ) text = "My email address is demo@lol.com, and my phone number is 1234567890" - output = guard.parse( - llm_output=text, - ) - assert output.error is not None + with pytest.raises(ValueError) as excinfo: + guard.parse( + llm_output=text, + ) + assert str(excinfo.value) == "`pii_entities` must be one of ['pii', 'spi']" def test_toxic_language(mocker): diff --git a/tests/integration_tests/validators/test_on_topic.py b/tests/integration_tests/validators/test_on_topic.py index 036af7afc..4ea718802 100644 --- a/tests/integration_tests/validators/test_on_topic.py +++ b/tests/integration_tests/validators/test_on_topic.py @@ -7,6 +7,8 @@ from guardrails.validators.on_topic import OnTopic +# TODO: Skip this is the CI. +# This test downloads the model which takes a substantial amount of space and time. class TestOnTopicIntegrationCPU(TestCase): def test_validate_valid_topic_cpu_disable_llm(self): validator = OnTopic( diff --git a/tests/unit_tests/test_guard.py b/tests/unit_tests/test_guard.py index 0516b1aa8..eefedb1b7 100644 --- a/tests/unit_tests/test_guard.py +++ b/tests/unit_tests/test_guard.py @@ -1,3 +1,4 @@ +from typing import Dict import openai import pytest from pydantic import BaseModel @@ -25,7 +26,7 @@ def validate(self, value, metadata): @pytest.mark.parametrize( - "spec,metadata", + "spec,metadata,error_message", [ ( """ @@ -36,6 +37,7 @@ def validate(self, value, metadata): """, {"required_key": "a"}, + "Missing required metadata keys: required_key" ), ( """ @@ -51,6 +53,7 @@ def validate(self, value, metadata): """, {"required_key": "a", "required_key2": "b"}, + "Missing required metadata keys: required_key2, required_key" ), ( """ @@ -72,12 +75,13 @@ def validate(self, value, metadata): """, {"required_key": "a"}, + "Missing required metadata keys: required_key" ), ], ) @pytest.mark.asyncio @pytest.mark.skipif(not OPENAI_VERSION.startswith("0"), reason="Only for OpenAI v0") -async def test_required_metadata(spec, metadata): +async def test_required_metadata(spec, metadata, error_message): guard = Guard.from_rail_string(spec) missing_keys = verify_metadata_requirements({}, guard.output_schema.root_datatype) @@ -89,17 +93,22 @@ async def test_required_metadata(spec, metadata): assert not_missing_keys == [] # test sync guard - response = guard.parse("{}") - assert response.error is not None + with pytest.raises(ValueError) as excinfo: + guard.parse("{}") + assert str(excinfo.value) == error_message response = guard.parse("{}", metadata=metadata, num_reasks=0) assert response.error is None # test async guard - response = await guard.parse( - "{}", llm_api=openai.ChatCompletion.acreate, num_reasks=0 - ) - assert response.error is not None + with pytest.raises(ValueError) as excinfo: + guard.parse("{}") + await guard.parse( + "{}", llm_api=openai.ChatCompletion.acreate, num_reasks=0 + ) + assert str(excinfo.value) == error_message + + response = await guard.parse( "{}", metadata=metadata, llm_api=openai.ChatCompletion.acreate, num_reasks=0 diff --git a/tests/unit_tests/test_validators.py b/tests/unit_tests/test_validators.py index 9f103a2a3..61a0b2bf9 100644 --- a/tests/unit_tests/test_validators.py +++ b/tests/unit_tests/test_validators.py @@ -423,6 +423,7 @@ def test_provenance_v1(mocker): assert output.validated_output == LLM_RESPONSE # 2. Setting the environment variable + openai_api_key_backup = os.environ["OPENAI_API_KEY"] os.environ["OPENAI_API_KEY"] = API_KEY output = string_guard.parse( llm_output=LLM_RESPONSE, @@ -438,6 +439,7 @@ def test_provenance_v1(mocker): api_base="https://api.openai.com", ) assert output.validated_output == LLM_RESPONSE + os.environ["OPENAI_API_KEY"] = openai_api_key_backup @pytest.mark.parametrize( @@ -675,14 +677,17 @@ class Pet(BaseModel): name: str = Field(description="a unique pet name") guard = Guard.from_pydantic(output_class=Pet, prompt=prompt) - response = guard.parse(output, num_reasks=0) if isinstance(expected_result, type) and issubclass(expected_result, Exception): - assert response.error is not None - assert response.error == "Something went wrong!" - elif isinstance(expected_result, FieldReAsk): - assert guard.history.first.iterations.first.reasks[0] == expected_result + with pytest.raises(type(expected_result)) as excinfo: + guard.parse(output, num_reasks=0) + + assert str(excinfo.value) == "Something went wrong!" else: - assert response.validated_output == expected_result + response = guard.parse(output, num_reasks=0) + if isinstance(expected_result, FieldReAsk): + assert guard.history.first.iterations.first.reasks[0] == expected_result + else: + assert response.validated_output == expected_result class Pet(BaseModel): @@ -940,51 +945,106 @@ def test_input_validation_fail(on_fail): @pytest.mark.parametrize( - "on_fail", + "on_fail, structured_prompt_error, structured_instructions_error, structured_message_history_error, unstructured_prompt_error, unstructured_instructions_error", [ - "reask", - "filter", - "refrain", - "exception", + # ( + # "reask", + # "Prompt validation failed: incorrect_value='What kind of pet should I get?\\n\\nJson Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", + # "Instructions validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", + # "Message history validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", + # "Prompt validation failed: incorrect_value='\\nThis is not two words\\n\\n\\nString Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This is')] path=None", + # "Instructions validation failed: incorrect_value='\\nThis also is not two words\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This also')] path=None" + # ), + # ( + # "filter", + # "Prompt validation failed", + # "Instructions validation failed", + # "Message history validation failed", + # "Prompt validation failed", + # "Instructions validation failed" + # ), + # ( + # "refrain", + # "Prompt validation failed", + # "Instructions validation failed", + # "Message history validation failed", + # "Prompt validation failed", + # "Instructions validation failed" + # ), + ( + "exception", + "Prompt validation failed", + "Instructions validation failed", + "Message history validation failed", + "Prompt validation failed", + "Instructions validation failed" + ), ], ) @pytest.mark.asyncio @pytest.mark.skipif(not OPENAI_VERSION.startswith("0"), reason="Not supported in v1") -async def test_input_validation_fail_async(on_fail): +async def test_input_validation_fail_async( + on_fail, + structured_prompt_error, + structured_instructions_error, + structured_message_history_error, + unstructured_prompt_error, + unstructured_instructions_error, +): # with_prompt_validation guard = Guard.from_pydantic(output_class=Pet).with_prompt_validation( validators=[TwoWords(on_fail=on_fail)] ) - await guard( - get_static_openai_acreate_func(), - prompt="What kind of pet should I get?", - ) + with pytest.raises(ValidatorError) as excinfo: + await guard( + get_static_openai_acreate_func(), + prompt="What kind of pet should I get?", + ) + print(" ") + print(str(excinfo.value)) + assert str(excinfo.value) == structured_prompt_error assert isinstance(guard.history.last.exception, ValidatorError) + assert guard.history.last.exception == excinfo.value + # with_instructions_validation guard = Guard.from_pydantic(output_class=Pet).with_instructions_validation( validators=[TwoWords(on_fail=on_fail)] ) - await guard( - get_static_openai_acreate_func(), - prompt="What kind of pet should I get and what should I name it?", - instructions="What kind of pet should I get?", - ) + with pytest.raises(ValidatorError) as excinfo: + await guard( + get_static_openai_acreate_func(), + prompt="What kind of pet should I get and what should I name it?", + instructions="What kind of pet should I get?", + ) + print(" ") + print(str(excinfo.value)) + assert str(excinfo.value) == structured_instructions_error assert isinstance(guard.history.last.exception, ValidatorError) + assert guard.history.last.exception == excinfo.value + + # with_msg_history_validation guard = Guard.from_pydantic(output_class=Pet).with_msg_history_validation( validators=[TwoWords(on_fail=on_fail)] ) - await guard( - get_static_openai_acreate_func(), - msg_history=[ - { - "role": "user", - "content": "What kind of pet should I get?", - } - ], - ) + with pytest.raises(ValidatorError) as excinfo: + await guard( + get_static_openai_acreate_func(), + msg_history=[ + { + "role": "user", + "content": "What kind of pet should I get?", + } + ], + ) + print(" ") + print(str(excinfo.value)) + assert str(excinfo.value) == structured_message_history_error assert isinstance(guard.history.last.exception, ValidatorError) + assert guard.history.last.exception == excinfo.value + + # rail prompt validation guard = Guard.from_rail_string( f""" @@ -1000,10 +1060,17 @@ async def test_input_validation_fail_async(on_fail): """ ) - await guard( - get_static_openai_acreate_func(), - ) + with pytest.raises(ValidatorError) as excinfo: + await guard( + get_static_openai_acreate_func(), + ) + print(" ") + print(str(excinfo.value)) + assert str(excinfo.value) == unstructured_prompt_error assert isinstance(guard.history.last.exception, ValidatorError) + assert guard.history.last.exception == excinfo.value + + # rail instructions validation guard = Guard.from_rail_string( f""" @@ -1022,10 +1089,15 @@ async def test_input_validation_fail_async(on_fail): """ ) - await guard( - get_static_openai_acreate_func(), - ) + with pytest.raises(ValidatorError) as excinfo: + await guard( + get_static_openai_acreate_func(), + ) + print(" ") + print(str(excinfo.value)) + assert str(excinfo.value) == unstructured_instructions_error assert isinstance(guard.history.last.exception, ValidatorError) + assert guard.history.last.exception == excinfo.value def test_input_validation_mismatch_raise(): From 5f5a5eb970fc8bfe9814f6c48f2d8e84673e231f Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Wed, 13 Dec 2023 16:10:07 -0600 Subject: [PATCH 02/20] fix tests, start lint --- guardrails/classes/history/call.py | 8 +- guardrails/datatypes.py | 4 +- guardrails/validators/pii_filter.py | 8 +- .../integration_tests/test_data_validation.py | 14 +- tests/integration_tests/test_datatypes.py | 6 +- .../test_embedding_openai.py | 8 +- tests/integration_tests/test_validators.py | 8 +- tests/unit_tests/test_guard.py | 13 +- tests/unit_tests/test_validators.py | 280 ++++++++++++------ 9 files changed, 228 insertions(+), 121 deletions(-) diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py index 07645adfc..9fef2bc9c 100644 --- a/guardrails/classes/history/call.py +++ b/guardrails/classes/history/call.py @@ -1,7 +1,6 @@ from typing import Dict, Optional, Union -from pydantic import PrivateAttr -from pydantic import Field +from pydantic import Field, PrivateAttr from rich.panel import Panel from rich.tree import Tree @@ -41,7 +40,9 @@ def __init__( ): iterations = iterations or Stack() inputs = inputs or CallInputs() - super().__init__(iterations=iterations, inputs=inputs, _exception=None) # type: ignore + super().__init__( # type: ignore + iterations=iterations, inputs=inputs, _exception=None + ) self.iterations = iterations self.inputs = inputs self._exception = None @@ -285,7 +286,6 @@ def error(self) -> Optional[str]: elif self.iterations.empty(): return None return self.iterations.last.error # type: ignore - @property def exception(self) -> Optional[Exception]: diff --git a/guardrails/datatypes.py b/guardrails/datatypes.py index 629c1219f..6ca697ee9 100644 --- a/guardrails/datatypes.py +++ b/guardrails/datatypes.py @@ -48,7 +48,9 @@ def verify_metadata_requirements( metadata, vars(datatype.children).values() ) missing_keys.update(nested_missing_keys) - return list(missing_keys) + missing_keys = list(missing_keys) + missing_keys.sort() + return missing_keys class DataType: diff --git a/guardrails/validators/pii_filter.py b/guardrails/validators/pii_filter.py index 38cd2a34b..3108f0316 100644 --- a/guardrails/validators/pii_filter.py +++ b/guardrails/validators/pii_filter.py @@ -108,20 +108,18 @@ def validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult: "in init or metadata." ) + pii_keys = list(self.PII_ENTITIES_MAP.keys()) # Check that pii_entities is a string OR list of strings if isinstance(pii_entities, str): # A key to the PII_ENTITIES_MAP entities_to_filter = self.PII_ENTITIES_MAP.get(pii_entities, None) if entities_to_filter is None: - raise ValueError( - f"`pii_entities` must be one of {list(self.PII_ENTITIES_MAP.keys())}" - ) + raise ValueError(f"`pii_entities` must be one of {pii_keys}") elif isinstance(pii_entities, list): entities_to_filter = pii_entities else: raise ValueError( - f"`pii_entities` must be one of {list(self.PII_ENTITIES_MAP.keys())}" - " or a list of strings." + f"`pii_entities` must be one of {pii_keys}" " or a list of strings." ) # Analyze the text, and anonymize it if there is PII diff --git a/tests/integration_tests/test_data_validation.py b/tests/integration_tests/test_data_validation.py index 226ac8f1d..82f1606f8 100644 --- a/tests/integration_tests/test_data_validation.py +++ b/tests/integration_tests/test_data_validation.py @@ -26,8 +26,18 @@ ), ('{"choice": {"action": "random_action"}}', False, True, False), ('{"choice": {"action": "fight", "fight_move": "random_move"}}', True, True, True), - ('{"choice": {"action": "flight", "random_key": "random_value"}}', False, True, False), - ('{"choice": {"action": "flight", "random_key": "random_value"}', False, True, True), + ( + '{"choice": {"action": "flight", "random_key": "random_value"}}', + False, + True, + False, + ), + ( + '{"choice": {"action": "flight", "random_key": "random_value"}', + False, + True, + True, + ), ] diff --git a/tests/integration_tests/test_datatypes.py b/tests/integration_tests/test_datatypes.py index 85bb1f099..e1083f78b 100644 --- a/tests/integration_tests/test_datatypes.py +++ b/tests/integration_tests/test_datatypes.py @@ -1,6 +1,6 @@ import pytest - from dateutil.parser import ParserError + from guardrails.guard import Guard @@ -64,7 +64,9 @@ def test_defaulted_date_parser(date_string: str): ("1697579939213", OverflowError), # Unix timestamp/milliseconds ], ) -def test_defaulted_date_parser_unsupported_values(date_string: str, error_type: Exception): +def test_defaulted_date_parser_unsupported_values( + date_string: str, error_type: Exception +): rail_spec = """ diff --git a/tests/integration_tests/test_embedding_openai.py b/tests/integration_tests/test_embedding_openai.py index 8fad453d6..eebf7bfa5 100644 --- a/tests/integration_tests/test_embedding_openai.py +++ b/tests/integration_tests/test_embedding_openai.py @@ -30,12 +30,11 @@ def __init__(self, data=None): def json(self): return {"data": self.data} - + def __getitem__(self, key: str): return getattr(self, key) - @pytest.mark.skipif( os.environ.get("OPENAI_API_KEY") in [None, "mocked"], reason="openai api key not set", @@ -73,12 +72,13 @@ def test__get_embedding(self, mocker): mock_create = None if OPENAI_VERSION.startswith("0"): mock_create = mocker.patch("openai.Embedding.create") - mock_create.return_value = MockResponse(data=[{"embedding": [1.0, 2.0, 3.0]}]) + mock_create.return_value = MockResponse( + data=[{"embedding": [1.0, 2.0, 3.0]}] + ) else: mock_create = mocker.patch("openai.resources.Embeddings.create") mock_create.return_value = MockResponse(data=[[1.0, 2.0, 3.0]]) - instance = OpenAIEmbedding(api_key="test_api_key") result = instance._get_embedding(["test text"]) assert result == [[1.0, 2.0, 3.0]] diff --git a/tests/integration_tests/test_validators.py b/tests/integration_tests/test_validators.py index a1907b91c..4cf22c745 100644 --- a/tests/integration_tests/test_validators.py +++ b/tests/integration_tests/test_validators.py @@ -86,7 +86,9 @@ def embed_function(text: str): guard.parse( llm_output=val, ) - assert str(excinfo.value) == "You must provide a list of previous values in metadata." + assert ( + str(excinfo.value) == "You must provide a list of previous values in metadata." + ) # 1.3 Test passing str prev values for int val # Should raise ValueError @@ -126,7 +128,9 @@ def embed_function(text: str): llm_output=val, metadata={"embed_function": embed_function}, ) - assert str(excinfo.value) == "You must provide a list of previous values in metadata." + assert ( + str(excinfo.value) == "You must provide a list of previous values in metadata." + ) # 2.3 Test passing int prev values for str val # Should raise ValueError diff --git a/tests/unit_tests/test_guard.py b/tests/unit_tests/test_guard.py index eefedb1b7..6fd515a50 100644 --- a/tests/unit_tests/test_guard.py +++ b/tests/unit_tests/test_guard.py @@ -1,4 +1,3 @@ -from typing import Dict import openai import pytest from pydantic import BaseModel @@ -37,7 +36,7 @@ def validate(self, value, metadata): """, {"required_key": "a"}, - "Missing required metadata keys: required_key" + "Missing required metadata keys: required_key", ), ( """ @@ -53,7 +52,7 @@ def validate(self, value, metadata): """, {"required_key": "a", "required_key2": "b"}, - "Missing required metadata keys: required_key2, required_key" + "Missing required metadata keys: required_key, required_key2", ), ( """ @@ -75,7 +74,7 @@ def validate(self, value, metadata): """, {"required_key": "a"}, - "Missing required metadata keys: required_key" + "Missing required metadata keys: required_key", ), ], ) @@ -103,13 +102,9 @@ async def test_required_metadata(spec, metadata, error_message): # test async guard with pytest.raises(ValueError) as excinfo: guard.parse("{}") - await guard.parse( - "{}", llm_api=openai.ChatCompletion.acreate, num_reasks=0 - ) + await guard.parse("{}", llm_api=openai.ChatCompletion.acreate, num_reasks=0) assert str(excinfo.value) == error_message - - response = await guard.parse( "{}", metadata=metadata, llm_api=openai.ChatCompletion.acreate, num_reasks=0 ) diff --git a/tests/unit_tests/test_validators.py b/tests/unit_tests/test_validators.py index 61a0b2bf9..813e65457 100644 --- a/tests/unit_tests/test_validators.py +++ b/tests/unit_tests/test_validators.py @@ -1,4 +1,5 @@ # noqa:W291 +import json import os from typing import Any, Dict, List @@ -678,9 +679,8 @@ class Pet(BaseModel): guard = Guard.from_pydantic(output_class=Pet, prompt=prompt) if isinstance(expected_result, type) and issubclass(expected_result, Exception): - with pytest.raises(type(expected_result)) as excinfo: + with pytest.raises(ValidatorError) as excinfo: guard.parse(output, num_reasks=0) - assert str(excinfo.value) == "Something went wrong!" else: response = guard.parse(output, num_reasks=0) @@ -694,7 +694,43 @@ class Pet(BaseModel): name: str = Field(description="a unique pet name") -def test_input_validation_fix(): +def test_input_validation_fix(mocker): + if OPENAI_VERSION.startswith("0"): + mock_openai = mocker.patch("openai.Completion.create") + mock_openai.return_value = { + "choices": [ + { + "text": json.dumps({"name": "Fluffy"}), + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 20, + }, + } + else: + from openai.types import Completion, CompletionChoice, CompletionUsage + + return Completion( + id="", + choices=[ + CompletionChoice( + finish_reason="stop", + index=0, + logprobs=None, + text=json.dumps({"name": "Fluffy"}), + ), + ], + created=0, + model="", + object="text_completion", + usage=CompletionUsage( + completion_tokens=20, + prompt_tokens=10, + total_tokens=30, + ), + ) + # fix returns an amended value for prompt/instructions validation, guard = Guard.from_pydantic(output_class=Pet).with_prompt_validation( validators=[TwoWords(on_fail="fix")] @@ -720,16 +756,19 @@ def test_input_validation_fix(): guard = Guard.from_pydantic(output_class=Pet).with_msg_history_validation( validators=[TwoWords(on_fail="fix")] ) - guard( - get_static_openai_create_func(), - msg_history=[ - { - "role": "user", - "content": "What kind of pet should I get?", - } - ], - ) + with pytest.raises(ValidatorError) as excinfo: + guard( + get_static_openai_create_func(), + msg_history=[ + { + "role": "user", + "content": "What kind of pet should I get?", + } + ], + ) + assert str(excinfo.value) == "Message history validation failed" assert isinstance(guard.history.first.exception, ValidatorError) + assert guard.history.first.exception == excinfo.value # rail prompt validation guard = Guard.from_rail_string( @@ -777,7 +816,20 @@ def test_input_validation_fix(): @pytest.mark.asyncio @pytest.mark.skipif(not OPENAI_VERSION.startswith("0"), reason="Not supported in v1") -async def test_async_input_validation_fix(): +async def test_async_input_validation_fix(mocker): + mock_openai = mocker.patch("openai.Completion.acreate") + mock_openai.return_value = { + "choices": [ + { + "text": json.dumps({"name": "Fluffy"}), + } + ], + "usage": { + "prompt_tokens": 10, + "completion_tokens": 20, + }, + } + # fix returns an amended value for prompt/instructions validation, guard = Guard.from_pydantic(output_class=Pet).with_prompt_validation( validators=[TwoWords(on_fail="fix")] @@ -787,6 +839,7 @@ async def test_async_input_validation_fix(): prompt="What kind of pet should I get?", ) assert guard.history.first.iterations.first.outputs.validation_output == "What kind" + guard = Guard.from_pydantic(output_class=Pet).with_instructions_validation( validators=[TwoWords(on_fail="fix")] ) @@ -803,16 +856,19 @@ async def test_async_input_validation_fix(): guard = Guard.from_pydantic(output_class=Pet).with_msg_history_validation( validators=[TwoWords(on_fail="fix")] ) - await guard( - get_static_openai_acreate_func(), - msg_history=[ - { - "role": "user", - "content": "What kind of pet should I get?", - } - ], - ) + with pytest.raises(ValidatorError) as excinfo: + await guard( + get_static_openai_acreate_func(), + msg_history=[ + { + "role": "user", + "content": "What kind of pet should I get?", + } + ], + ) + assert str(excinfo.value) == "Message history validation failed" assert isinstance(guard.history.first.exception, ValidatorError) + assert guard.history.first.exception == excinfo.value # rail prompt validation guard = Guard.from_rail_string( @@ -859,48 +915,95 @@ async def test_async_input_validation_fix(): @pytest.mark.parametrize( - "on_fail", + "on_fail, structured_prompt_error, structured_instructions_error, structured_message_history_error, unstructured_prompt_error, unstructured_instructions_error", [ - "reask", - "filter", - "refrain", - "exception", + ( + "reask", + "Prompt validation failed: incorrect_value='What kind of pet should I get?\\n\\nJson Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", + "Instructions validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", + "Message history validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", + "Prompt validation failed: incorrect_value='\\nThis is not two words\\n\\n\\nString Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This is')] path=None", + "Instructions validation failed: incorrect_value='\\nThis also is not two words\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This also')] path=None", + ), + ( + "filter", + "Prompt validation failed", + "Instructions validation failed", + "Message history validation failed", + "Prompt validation failed", + "Instructions validation failed", + ), + ( + "refrain", + "Prompt validation failed", + "Instructions validation failed", + "Message history validation failed", + "Prompt validation failed", + "Instructions validation failed", + ), + ( + "exception", + "Validation failed for field with errors: must be exactly two words", + "Validation failed for field with errors: must be exactly two words", + "Validation failed for field with errors: must be exactly two words", + "Validation failed for field with errors: must be exactly two words", + "Validation failed for field with errors: must be exactly two words", + ), ], ) -def test_input_validation_fail(on_fail): +def test_input_validation_fail( + on_fail, + structured_prompt_error, + structured_instructions_error, + structured_message_history_error, + unstructured_prompt_error, + unstructured_instructions_error, +): # with_prompt_validation guard = Guard.from_pydantic(output_class=Pet).with_prompt_validation( validators=[TwoWords(on_fail=on_fail)] ) - guard( - get_static_openai_create_func(), - prompt="What kind of pet should I get?", - ) + with pytest.raises(ValidatorError) as excinfo: + guard( + get_static_openai_create_func(), + prompt="What kind of pet should I get?", + ) + assert str(excinfo.value) == structured_prompt_error assert isinstance(guard.history.last.exception, ValidatorError) + assert guard.history.last.exception == excinfo.value + # with_instructions_validation guard = Guard.from_pydantic(output_class=Pet).with_instructions_validation( validators=[TwoWords(on_fail=on_fail)] ) - guard( - get_static_openai_create_func(), - prompt="What kind of pet should I get and what should I name it?", - instructions="What kind of pet should I get?", - ) + with pytest.raises(ValidatorError) as excinfo: + guard( + get_static_openai_create_func(), + prompt="What kind of pet should I get and what should I name it?", + instructions="What kind of pet should I get?", + ) + assert str(excinfo.value) == structured_instructions_error assert isinstance(guard.history.last.exception, ValidatorError) + assert guard.history.last.exception == excinfo.value + # with_msg_history_validation guard = Guard.from_pydantic(output_class=Pet).with_msg_history_validation( validators=[TwoWords(on_fail=on_fail)] ) - guard( - get_static_openai_create_func(), - msg_history=[ - { - "role": "user", - "content": "What kind of pet should I get?", - } - ], - ) + with pytest.raises(ValidatorError) as excinfo: + guard( + get_static_openai_create_func(), + msg_history=[ + { + "role": "user", + "content": "What kind of pet should I get?", + } + ], + ) + assert str(excinfo.value) == structured_message_history_error assert isinstance(guard.history.last.exception, ValidatorError) + assert guard.history.last.exception == excinfo.value + # rail prompt validation guard = Guard.from_rail_string( f""" @@ -916,10 +1019,14 @@ def test_input_validation_fail(on_fail): """ ) - guard( - get_static_openai_create_func(), - ) + with pytest.raises(ValidatorError) as excinfo: + guard( + get_static_openai_create_func(), + ) + assert str(excinfo.value) == unstructured_prompt_error assert isinstance(guard.history.last.exception, ValidatorError) + assert guard.history.last.exception == excinfo.value + # rail instructions validation guard = Guard.from_rail_string( f""" @@ -938,46 +1045,49 @@ def test_input_validation_fail(on_fail): """ ) - guard( - get_static_openai_create_func(), - ) + with pytest.raises(ValidatorError) as excinfo: + guard( + get_static_openai_create_func(), + ) + assert str(excinfo.value) == unstructured_instructions_error assert isinstance(guard.history.last.exception, ValidatorError) + assert guard.history.last.exception == excinfo.value @pytest.mark.parametrize( "on_fail, structured_prompt_error, structured_instructions_error, structured_message_history_error, unstructured_prompt_error, unstructured_instructions_error", [ - # ( - # "reask", - # "Prompt validation failed: incorrect_value='What kind of pet should I get?\\n\\nJson Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", - # "Instructions validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", - # "Message history validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", - # "Prompt validation failed: incorrect_value='\\nThis is not two words\\n\\n\\nString Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This is')] path=None", - # "Instructions validation failed: incorrect_value='\\nThis also is not two words\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This also')] path=None" - # ), - # ( - # "filter", - # "Prompt validation failed", - # "Instructions validation failed", - # "Message history validation failed", - # "Prompt validation failed", - # "Instructions validation failed" - # ), - # ( - # "refrain", - # "Prompt validation failed", - # "Instructions validation failed", - # "Message history validation failed", - # "Prompt validation failed", - # "Instructions validation failed" - # ), ( - "exception", + "reask", + "Prompt validation failed: incorrect_value='What kind of pet should I get?\\n\\nJson Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", + "Instructions validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", + "Message history validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", + "Prompt validation failed: incorrect_value='\\nThis is not two words\\n\\n\\nString Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This is')] path=None", + "Instructions validation failed: incorrect_value='\\nThis also is not two words\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This also')] path=None", + ), + ( + "filter", "Prompt validation failed", "Instructions validation failed", "Message history validation failed", "Prompt validation failed", - "Instructions validation failed" + "Instructions validation failed", + ), + ( + "refrain", + "Prompt validation failed", + "Instructions validation failed", + "Message history validation failed", + "Prompt validation failed", + "Instructions validation failed", + ), + ( + "exception", + "Validation failed for field with errors: must be exactly two words", + "Validation failed for field with errors: must be exactly two words", + "Validation failed for field with errors: must be exactly two words", + "Validation failed for field with errors: must be exactly two words", + "Validation failed for field with errors: must be exactly two words", ), ], ) @@ -1000,13 +1110,10 @@ async def test_input_validation_fail_async( get_static_openai_acreate_func(), prompt="What kind of pet should I get?", ) - print(" ") - print(str(excinfo.value)) assert str(excinfo.value) == structured_prompt_error assert isinstance(guard.history.last.exception, ValidatorError) assert guard.history.last.exception == excinfo.value - # with_instructions_validation guard = Guard.from_pydantic(output_class=Pet).with_instructions_validation( validators=[TwoWords(on_fail=on_fail)] @@ -1017,13 +1124,10 @@ async def test_input_validation_fail_async( prompt="What kind of pet should I get and what should I name it?", instructions="What kind of pet should I get?", ) - print(" ") - print(str(excinfo.value)) assert str(excinfo.value) == structured_instructions_error assert isinstance(guard.history.last.exception, ValidatorError) assert guard.history.last.exception == excinfo.value - # with_msg_history_validation guard = Guard.from_pydantic(output_class=Pet).with_msg_history_validation( validators=[TwoWords(on_fail=on_fail)] @@ -1038,13 +1142,10 @@ async def test_input_validation_fail_async( } ], ) - print(" ") - print(str(excinfo.value)) assert str(excinfo.value) == structured_message_history_error assert isinstance(guard.history.last.exception, ValidatorError) assert guard.history.last.exception == excinfo.value - # rail prompt validation guard = Guard.from_rail_string( f""" @@ -1064,13 +1165,10 @@ async def test_input_validation_fail_async( await guard( get_static_openai_acreate_func(), ) - print(" ") - print(str(excinfo.value)) assert str(excinfo.value) == unstructured_prompt_error assert isinstance(guard.history.last.exception, ValidatorError) assert guard.history.last.exception == excinfo.value - # rail instructions validation guard = Guard.from_rail_string( f""" @@ -1093,8 +1191,6 @@ async def test_input_validation_fail_async( await guard( get_static_openai_acreate_func(), ) - print(" ") - print(str(excinfo.value)) assert str(excinfo.value) == unstructured_instructions_error assert isinstance(guard.history.last.exception, ValidatorError) assert guard.history.last.exception == excinfo.value From 40777905d9f7952f320b78d2a2ea9c81ee5c5e83 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Thu, 14 Dec 2023 08:46:09 -0600 Subject: [PATCH 03/20] lint and type --- guardrails/classes/history/call.py | 4 ++-- guardrails/validators/on_topic.py | 2 -- tests/unit_tests/test_validators.py | 34 +++++++++++++++++++---------- 3 files changed, 24 insertions(+), 16 deletions(-) diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py index 9fef2bc9c..b9d3d3a67 100644 --- a/guardrails/classes/history/call.py +++ b/guardrails/classes/history/call.py @@ -41,11 +41,11 @@ def __init__( iterations = iterations or Stack() inputs = inputs or CallInputs() super().__init__( # type: ignore - iterations=iterations, inputs=inputs, _exception=None + iterations=iterations, inputs=inputs, _exception=None # type: ignore ) self.iterations = iterations self.inputs = inputs - self._exception = None + self._exception = None # type: ignore @property def prompt(self) -> Optional[str]: diff --git a/guardrails/validators/on_topic.py b/guardrails/validators/on_topic.py index eeeb54d17..5b4c01bc7 100644 --- a/guardrails/validators/on_topic.py +++ b/guardrails/validators/on_topic.py @@ -242,8 +242,6 @@ def validate(self, value: str, metadata: Dict[str, Any]) -> ValidationResult: candidate_topics = valid_topics.union(invalid_topics) # Check which model(s) to use - print("self._disable_classifier: ", self._disable_classifier) - print("self._disable_llm: ", self._disable_llm) if self._disable_classifier and self._disable_llm: # Error, no model set raise ValueError("Either classifier or llm must be enabled.") elif ( diff --git a/tests/unit_tests/test_validators.py b/tests/unit_tests/test_validators.py index 813e65457..317ad3671 100644 --- a/tests/unit_tests/test_validators.py +++ b/tests/unit_tests/test_validators.py @@ -915,15 +915,20 @@ async def test_async_input_validation_fix(mocker): @pytest.mark.parametrize( - "on_fail, structured_prompt_error, structured_instructions_error, structured_message_history_error, unstructured_prompt_error, unstructured_instructions_error", + "on_fail," + "structured_prompt_error," + "structured_instructions_error," + "structured_message_history_error," + "unstructured_prompt_error," + "unstructured_instructions_error", [ ( "reask", - "Prompt validation failed: incorrect_value='What kind of pet should I get?\\n\\nJson Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", - "Instructions validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", - "Message history validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", - "Prompt validation failed: incorrect_value='\\nThis is not two words\\n\\n\\nString Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This is')] path=None", - "Instructions validation failed: incorrect_value='\\nThis also is not two words\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This also')] path=None", + "Prompt validation failed: incorrect_value='What kind of pet should I get?\\n\\nJson Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", # noqa + "Instructions validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", # noqa + "Message history validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", # noqa + "Prompt validation failed: incorrect_value='\\nThis is not two words\\n\\n\\nString Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This is')] path=None", # noqa + "Instructions validation failed: incorrect_value='\\nThis also is not two words\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This also')] path=None", # noqa ), ( "filter", @@ -1055,15 +1060,20 @@ def test_input_validation_fail( @pytest.mark.parametrize( - "on_fail, structured_prompt_error, structured_instructions_error, structured_message_history_error, unstructured_prompt_error, unstructured_instructions_error", + "on_fail," + "structured_prompt_error," + "structured_instructions_error," + "structured_message_history_error," + "unstructured_prompt_error," + "unstructured_instructions_error", [ ( "reask", - "Prompt validation failed: incorrect_value='What kind of pet should I get?\\n\\nJson Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", - "Instructions validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", - "Message history validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", - "Prompt validation failed: incorrect_value='\\nThis is not two words\\n\\n\\nString Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This is')] path=None", - "Instructions validation failed: incorrect_value='\\nThis also is not two words\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This also')] path=None", + "Prompt validation failed: incorrect_value='What kind of pet should I get?\\n\\nJson Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", # noqa + "Instructions validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", # noqa + "Message history validation failed: incorrect_value='What kind of pet should I get?' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='What kind')] path=None", # noqa + "Prompt validation failed: incorrect_value='\\nThis is not two words\\n\\n\\nString Output:\\n\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This is')] path=None", # noqa + "Instructions validation failed: incorrect_value='\\nThis also is not two words\\n' fail_results=[FailResult(outcome='fail', metadata=None, error_message='must be exactly two words', fix_value='This also')] path=None", # noqa ), ( "filter", From dd96f289fd7bea78988ae5d31e2abdfefddc7404 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Thu, 14 Dec 2023 08:47:17 -0600 Subject: [PATCH 04/20] remove prints --- guardrails/run.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/guardrails/run.py b/guardrails/run.py index b02f4e884..5043f2e55 100644 --- a/guardrails/run.py +++ b/guardrails/run.py @@ -196,13 +196,9 @@ def __call__(self, call_log: Call, prompt_params: Optional[Dict] = None) -> Call ) except UserFacingException as e: call_log.exception = e.original_exception - # import traceback - # traceback.print_exception(e) raise e.original_exception except Exception as e: call_log.exception = e - # import traceback - # traceback.print_exception(e) raise e return call_log @@ -314,8 +310,6 @@ def step( error_message = str(e) iteration.outputs.error = error_message iteration.outputs.exception = e - # import traceback - # traceback.print_exception(e) raise e return iteration @@ -778,13 +772,9 @@ async def async_run( ) except UserFacingException as e: call_log.exception = e.original_exception - # import traceback - # traceback.print_exception(e) raise e.original_exception except Exception as e: call_log.exception = e - # import traceback - # traceback.print_exception(e) raise e return call_log @@ -896,8 +886,6 @@ async def async_step( error_message = str(e) iteration.outputs.error = error_message iteration.outputs.exception = e - # import traceback - # traceback.print_exception(e) raise e return iteration From efbbbda3072f4e5482c2d8789afb192197abebb2 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Thu, 14 Dec 2023 09:53:48 -0600 Subject: [PATCH 05/20] fix openai api_base in validator --- guardrails/validators/on_topic.py | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/guardrails/validators/on_topic.py b/guardrails/validators/on_topic.py index 5b4c01bc7..003178228 100644 --- a/guardrails/validators/on_topic.py +++ b/guardrails/validators/on_topic.py @@ -2,7 +2,6 @@ import json from typing import Any, Callable, Dict, List, Optional, Tuple, Union -import openai from tenacity import retry, stop_after_attempt, wait_random_exponential from transformers import pipeline @@ -126,7 +125,7 @@ def get_topic_llm(self, text: str, candidate_topics: List[str]) -> ValidationRes topic = json.loads(response)["topic"] return self.verify_topic(topic) - def set_client(self): + def get_client_args(self) -> Tuple[Optional[str], Optional[str]]: kwargs = {} context_copy = contextvars.copy_context() for key, context_var in context_copy.items(): @@ -137,13 +136,7 @@ def set_client(self): api_key = kwargs.get("api_key") api_base = kwargs.get("api_base") - # Set the OpenAI API key - if api_key: # Check if set when calling guard() or parse() - openai.api_key = api_key - - # Set the OpenAI API base if specified - if api_base: - openai.api_base = api_base + return (api_key, api_base) # todo: extract some of these similar methods into a base class w provenance @retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(5)) @@ -185,7 +178,8 @@ def set_callable(self, llm_callable: Union[str, Callable, None]) -> None: ) def openai_callable(text: str, topics: List[str]) -> str: - response = OpenAIClient().create_chat_completion( + api_key, api_base = self.get_client_args() + response = OpenAIClient(api_key, api_base).create_chat_completion( model=llm_callable, messages=[ { @@ -247,10 +241,8 @@ def validate(self, value: str, metadata: Dict[str, Any]) -> ValidationResult: elif ( not self._disable_classifier and not self._disable_llm ): # Use ensemble (Zero-Shot + Ensemble) - self.set_client() return self.get_topic_ensemble(value, list(candidate_topics)) elif self._disable_classifier and not self._disable_llm: # Use only LLM - self.set_client() return self.get_topic_llm(value, list(candidate_topics)) # Use only Zero-Shot From b562329e095e53e0284394dea6d83f3352c22a81 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Thu, 14 Dec 2023 10:59:53 -0600 Subject: [PATCH 06/20] work around pydantic v1 limitations --- guardrails/classes/history/call.py | 15 +++++++-------- guardrails/run.py | 12 ++++++++---- tests/unit_tests/test_validators.py | 2 +- 3 files changed, 16 insertions(+), 13 deletions(-) diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py index b9d3d3a67..a9fdf12bf 100644 --- a/guardrails/classes/history/call.py +++ b/guardrails/classes/history/call.py @@ -29,7 +29,7 @@ class Call(ArbitraryModel): inputs: CallInputs = Field( description="The inputs as passed in to Guard.__call__ or Guard.parse" ) - _exception: Exception = PrivateAttr() + _exception: Optional[Exception] = PrivateAttr() # Prevent Pydantic from changing our types # Without this, Pydantic casts iterations to a list @@ -37,15 +37,18 @@ def __init__( self, iterations: Optional[Stack[Iteration]] = None, inputs: Optional[CallInputs] = None, + exception: Optional[Exception] = None, ): iterations = iterations or Stack() inputs = inputs or CallInputs() - super().__init__( # type: ignore - iterations=iterations, inputs=inputs, _exception=None # type: ignore + super().__init__( + iterations=iterations, # type: ignore + inputs=inputs, # type: ignore + _exception=exception, # type: ignore ) self.iterations = iterations self.inputs = inputs - self._exception = None # type: ignore + self._exception = exception @property def prompt(self) -> Optional[str]: @@ -296,10 +299,6 @@ def exception(self) -> Optional[Exception]: return None return self.iterations.last.exception # type: ignore - @exception.setter - def exception(self, value: Exception) -> None: - self._exception = value - @property def failed_validations(self) -> Stack[ValidatorLogs]: """The validator logs for any validations that failed during the diff --git a/guardrails/run.py b/guardrails/run.py index 5043f2e55..11a9d2d56 100644 --- a/guardrails/run.py +++ b/guardrails/run.py @@ -195,10 +195,12 @@ def __call__(self, call_log: Call, prompt_params: Optional[Dict] = None) -> Call include_instructions=include_instructions, ) except UserFacingException as e: - call_log.exception = e.original_exception + # Because Pydantic v1 doesn't respect property setters + call_log._exception = e.original_exception raise e.original_exception except Exception as e: - call_log.exception = e + # Because Pydantic v1 doesn't respect property setters + call_log._exception = e raise e return call_log @@ -771,10 +773,12 @@ async def async_run( prompt_params=prompt_params, ) except UserFacingException as e: - call_log.exception = e.original_exception + # Because Pydantic v1 doesn't respect property setters + call_log._exception = e.original_exception raise e.original_exception except Exception as e: - call_log.exception = e + # Because Pydantic v1 doesn't respect property setters + call_log._exception = e raise e return call_log diff --git a/tests/unit_tests/test_validators.py b/tests/unit_tests/test_validators.py index 317ad3671..ca1d70cd9 100644 --- a/tests/unit_tests/test_validators.py +++ b/tests/unit_tests/test_validators.py @@ -424,7 +424,7 @@ def test_provenance_v1(mocker): assert output.validated_output == LLM_RESPONSE # 2. Setting the environment variable - openai_api_key_backup = os.environ["OPENAI_API_KEY"] + openai_api_key_backup = os.environ.get("OPENAI_API_KEY") os.environ["OPENAI_API_KEY"] = API_KEY output = string_guard.parse( llm_output=LLM_RESPONSE, From 06f94364d2cab28ec5f846a77872ae965fc34440 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Thu, 14 Dec 2023 11:13:25 -0600 Subject: [PATCH 07/20] del vs assign none --- tests/unit_tests/test_validators.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/unit_tests/test_validators.py b/tests/unit_tests/test_validators.py index ca1d70cd9..920bfe8f2 100644 --- a/tests/unit_tests/test_validators.py +++ b/tests/unit_tests/test_validators.py @@ -440,7 +440,10 @@ def test_provenance_v1(mocker): api_base="https://api.openai.com", ) assert output.validated_output == LLM_RESPONSE - os.environ["OPENAI_API_KEY"] = openai_api_key_backup + if openai_api_key_backup is not None: + os.environ["OPENAI_API_KEY"] = openai_api_key_backup + else: + del os.environ["OPENAI_API_KEY"] @pytest.mark.parametrize( From e94c634b8549487609205bad1bedc2cb0af60238 Mon Sep 17 00:00:00 2001 From: zsimjee Date: Mon, 11 Dec 2023 17:11:55 -0800 Subject: [PATCH 08/20] index on topic nb in docs --- docs/examples/response_is_on_topic.ipynb | 13 +++++++------ mkdocs.yml | 1 + 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/examples/response_is_on_topic.ipynb b/docs/examples/response_is_on_topic.ipynb index cf3be94e3..4fc393d1b 100644 --- a/docs/examples/response_is_on_topic.ipynb +++ b/docs/examples/response_is_on_topic.ipynb @@ -14,17 +14,18 @@ "This validator checks if a text is related with a topic. Using a list of valid topics (which can include one or many) and optionally a list of invalid topics, it validates that the text's main topic is one of the valid ones. If none of the valid topics are relevant, the topic 'Other' will be considered as the most relevant one and the validator will fail.\n", "\n", "The validator supports 3 different variants:\n", - "- Using an ensemble of Zero-Shot classifier + LLM fallback: if the original classification score is less than 0.5, an LLM is used to classify the main topic. This is the default behavior, setting `disable_classifier = False` and `disable_llm = False`.\n", - "- Using just a Zero-Shot classifier to get the main topic (`disable_classifier = False` and `disable_llm = True`).\n", - "- Using just an LLM to classify the main topic (`disable_classifier = True` and `disable_llm = False`).\n", + "\n", + "1. Using an ensemble of Zero-Shot classifier + LLM fallback: if the original classification score is less than 0.5, an LLM is used to classify the main topic. This is the default behavior, setting `disable_classifier = False` and `disable_llm = False`.\n", + "2. Using just a Zero-Shot classifier to get the main topic (`disable_classifier = False` and `disable_llm = True`).\n", + "3. Using just an LLM to classify the main topic (`disable_classifier = True` and `disable_llm = False`).\n", "\n", "To use the LLM, you can pass in a name of any OpenAI ChatCompletion model like `gpt-3.5-turbo` or `gpt-4` as the `llm_callable`, or pass in a callable that handles LLM calls. This callable can use any LLM, that you define. For simplicity purposes, we show here a demo of using OpenAI's gpt-3.5-turbo model.\n", "\n", "To use the OpenAI API, you have 3 options:\n", "\n", - "- Set the OPENAI_API_KEY environment variable: os.environ[\"OPENAI_API_KEY\"] = \"\"\n", - "- Set the OPENAI_API_KEY using openai.api_key=\"\"\n", - "- Pass the api_key as a parameter to the parse function as done below, in this example" + "1. Set the OPENAI_API_KEY environment variable: os.environ[\"OPENAI_API_KEY\"] = \"\"\n", + "2. Set the OPENAI_API_KEY using openai.api_key=\"\"\n", + "3. Pass the api_key as a parameter to the parse function" ] }, { diff --git a/mkdocs.yml b/mkdocs.yml index 55cd46c62..c0467391b 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -63,6 +63,7 @@ nav: - 'Check whether a value is similar to a set of other values': examples/value_within_distribution.ipynb - 'Using GuardrailsOutputParser in LlamaIndex': examples/llamaindex-output-parsing.ipynb - 'Check if a competitor is named': examples/competitors_check.ipynb + - 'Assure that responses are on topic': examples/response_is_on_topic.ipynb - 'Integrations': - 'Azure OpenAI': integrations/azure_openai.ipynb - 'OpenAI Functions': integrations/openai_functions.ipynb From 2609cef892443435334289cf02e9afd7b20f0846 Mon Sep 17 00:00:00 2001 From: zsimjee Date: Tue, 12 Dec 2023 10:00:31 -0800 Subject: [PATCH 09/20] add 'other' to invalid topics in every case --- guardrails/validators/on_topic.py | 6 ++++-- tests/integration_tests/validators/test_on_topic.py | 1 - 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/guardrails/validators/on_topic.py b/guardrails/validators/on_topic.py index 003178228..15c28170b 100644 --- a/guardrails/validators/on_topic.py +++ b/guardrails/validators/on_topic.py @@ -214,7 +214,9 @@ def get_topic_zero_shot( score = result["scores"][0] # type: ignore return topic, score # type: ignore - def validate(self, value: str, metadata: Dict[str, Any]) -> ValidationResult: + def validate( + self, value: str, metadata: Optional[Dict[str, Any]] + ) -> ValidationResult: valid_topics = set(self._valid_topics) invalid_topics = set(self._invalid_topics) @@ -230,7 +232,7 @@ def validate(self, value: str, metadata: Dict[str, Any]) -> ValidationResult: # Add 'other' to the invalid topics list if "other" not in invalid_topics: - self._invalid_topics.append("other") + invalid_topics.add("other") # Combine valid and invalid topics candidate_topics = valid_topics.union(invalid_topics) diff --git a/tests/integration_tests/validators/test_on_topic.py b/tests/integration_tests/validators/test_on_topic.py index 4ea718802..c04ced5fe 100644 --- a/tests/integration_tests/validators/test_on_topic.py +++ b/tests/integration_tests/validators/test_on_topic.py @@ -26,7 +26,6 @@ def test_validate_invalid_topic_cpu_disable_llm(self): valid_topics=["sports", "politics"], disable_classifier=False, disable_llm=True, - model_threshold=0.6, ) text = "This is an article about music." expected_result = FailResult(error_message="Most relevant topic is other.") From fdd45a85c584bd61830f432f523add8a95ce6325 Mon Sep 17 00:00:00 2001 From: Karan Acharya Date: Tue, 12 Dec 2023 16:09:14 -0500 Subject: [PATCH 10/20] New version commit --- poetry.lock | 219 ++++++++++++++++++++++++------------------------- pyproject.toml | 2 +- 2 files changed, 107 insertions(+), 114 deletions(-) diff --git a/poetry.lock b/poetry.lock index de2d26a81..db9da678e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -239,18 +239,17 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte [[package]] name = "babel" -version = "2.13.1" +version = "2.14.0" description = "Internationalization utilities" optional = false python-versions = ">=3.7" files = [ - {file = "Babel-2.13.1-py3-none-any.whl", hash = "sha256:7077a4984b02b6727ac10f1f7294484f737443d7e2e66c5e4380e41a3ae0b4ed"}, - {file = "Babel-2.13.1.tar.gz", hash = "sha256:33e0952d7dd6374af8dbf6768cc4ddf3ccfefc244f9986d4074704f2fbd18900"}, + {file = "Babel-2.14.0-py3-none-any.whl", hash = "sha256:efb1a25b7118e67ce3a259bed20545c29cb68be8ad2c784c83689981b7a57287"}, + {file = "Babel-2.14.0.tar.gz", hash = "sha256:6919867db036398ba21eb5c7a0f6b28ab8cbc3ae7a73a44ebe34ae74a4e7d363"}, ] [package.dependencies] pytz = {version = ">=2015.7", markers = "python_version < \"3.9\""} -setuptools = {version = "*", markers = "python_version >= \"3.12\""} [package.extras] dev = ["freezegun (>=1.0,<2.0)", "pytest (>=6.0)", "pytest-cov"] @@ -964,13 +963,13 @@ word-list = ["pyahocorasick"] [[package]] name = "distlib" -version = "0.3.7" +version = "0.3.8" description = "Distribution utilities" optional = false python-versions = "*" files = [ - {file = "distlib-0.3.7-py2.py3-none-any.whl", hash = "sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057"}, - {file = "distlib-0.3.7.tar.gz", hash = "sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8"}, + {file = "distlib-0.3.8-py2.py3-none-any.whl", hash = "sha256:034db59a0b96f8ca18035f36290806a9a6e6bd9d1ff91e45a7f172eb17e51784"}, + {file = "distlib-0.3.8.tar.gz", hash = "sha256:1530ea13e350031b6312d8580ddb6b27a104275a31106523b8f123787f494f64"}, ] [[package]] @@ -1251,13 +1250,13 @@ files = [ [[package]] name = "fsspec" -version = "2023.12.1" +version = "2023.12.2" description = "File-system specification" optional = true python-versions = ">=3.8" files = [ - {file = "fsspec-2023.12.1-py3-none-any.whl", hash = "sha256:6271f1d3075a378bfe432f6f42bf7e1d2a6ba74f78dd9b512385474c579146a0"}, - {file = "fsspec-2023.12.1.tar.gz", hash = "sha256:c4da01a35ac65c853f833e43f67802c25213f560820d54ddf248f92eddd5e990"}, + {file = "fsspec-2023.12.2-py3-none-any.whl", hash = "sha256:d800d87f72189a745fa3d6b033b9dc4a34ad069f60ca60b943a63599f5501960"}, + {file = "fsspec-2023.12.2.tar.gz", hash = "sha256:8548d39e8810b59c38014934f6b31e57f40c1b20f911f4cc2b85389c7e9bf0cb"}, ] [package.dependencies] @@ -1338,68 +1337,69 @@ test = ["black", "coverage[toml]", "ddt (>=1.1.1,!=1.4.3)", "mock", "mypy", "pre [[package]] name = "greenlet" -version = "3.0.1" +version = "3.0.2" description = "Lightweight in-process concurrent programming" optional = true python-versions = ">=3.7" files = [ - {file = "greenlet-3.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:f89e21afe925fcfa655965ca8ea10f24773a1791400989ff32f467badfe4a064"}, - {file = "greenlet-3.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:28e89e232c7593d33cac35425b58950789962011cc274aa43ef8865f2e11f46d"}, - {file = "greenlet-3.0.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b8ba29306c5de7717b5761b9ea74f9c72b9e2b834e24aa984da99cbfc70157fd"}, - {file = "greenlet-3.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:19bbdf1cce0346ef7341705d71e2ecf6f41a35c311137f29b8a2dc2341374565"}, - {file = "greenlet-3.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:599daf06ea59bfedbec564b1692b0166a0045f32b6f0933b0dd4df59a854caf2"}, - {file = "greenlet-3.0.1-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b641161c302efbb860ae6b081f406839a8b7d5573f20a455539823802c655f63"}, - {file = "greenlet-3.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d57e20ba591727da0c230ab2c3f200ac9d6d333860d85348816e1dca4cc4792e"}, - {file = "greenlet-3.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5805e71e5b570d490938d55552f5a9e10f477c19400c38bf1d5190d760691846"}, - {file = "greenlet-3.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:52e93b28db27ae7d208748f45d2db8a7b6a380e0d703f099c949d0f0d80b70e9"}, - {file = "greenlet-3.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:f7bfb769f7efa0eefcd039dd19d843a4fbfbac52f1878b1da2ed5793ec9b1a65"}, - {file = "greenlet-3.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:91e6c7db42638dc45cf2e13c73be16bf83179f7859b07cfc139518941320be96"}, - {file = "greenlet-3.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1757936efea16e3f03db20efd0cd50a1c86b06734f9f7338a90c4ba85ec2ad5a"}, - {file = "greenlet-3.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:19075157a10055759066854a973b3d1325d964d498a805bb68a1f9af4aaef8ec"}, - {file = "greenlet-3.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9d21aaa84557d64209af04ff48e0ad5e28c5cca67ce43444e939579d085da72"}, - {file = "greenlet-3.0.1-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:2847e5d7beedb8d614186962c3d774d40d3374d580d2cbdab7f184580a39d234"}, - {file = "greenlet-3.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:97e7ac860d64e2dcba5c5944cfc8fa9ea185cd84061c623536154d5a89237884"}, - {file = "greenlet-3.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:b2c02d2ad98116e914d4f3155ffc905fd0c025d901ead3f6ed07385e19122c94"}, - {file = "greenlet-3.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:22f79120a24aeeae2b4471c711dcf4f8c736a2bb2fabad2a67ac9a55ea72523c"}, - {file = "greenlet-3.0.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:100f78a29707ca1525ea47388cec8a049405147719f47ebf3895e7509c6446aa"}, - {file = "greenlet-3.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:60d5772e8195f4e9ebf74046a9121bbb90090f6550f81d8956a05387ba139353"}, - {file = "greenlet-3.0.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:daa7197b43c707462f06d2c693ffdbb5991cbb8b80b5b984007de431493a319c"}, - {file = "greenlet-3.0.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ea6b8aa9e08eea388c5f7a276fabb1d4b6b9d6e4ceb12cc477c3d352001768a9"}, - {file = "greenlet-3.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d11ebbd679e927593978aa44c10fc2092bc454b7d13fdc958d3e9d508aba7d0"}, - {file = "greenlet-3.0.1-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:dbd4c177afb8a8d9ba348d925b0b67246147af806f0b104af4d24f144d461cd5"}, - {file = "greenlet-3.0.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:20107edf7c2c3644c67c12205dc60b1bb11d26b2610b276f97d666110d1b511d"}, - {file = "greenlet-3.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8bef097455dea90ffe855286926ae02d8faa335ed8e4067326257cb571fc1445"}, - {file = "greenlet-3.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:b2d3337dcfaa99698aa2377c81c9ca72fcd89c07e7eb62ece3f23a3fe89b2ce4"}, - {file = "greenlet-3.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:80ac992f25d10aaebe1ee15df45ca0d7571d0f70b645c08ec68733fb7a020206"}, - {file = "greenlet-3.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:337322096d92808f76ad26061a8f5fccb22b0809bea39212cd6c406f6a7060d2"}, - {file = "greenlet-3.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b9934adbd0f6e476f0ecff3c94626529f344f57b38c9a541f87098710b18af0a"}, - {file = "greenlet-3.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dc4d815b794fd8868c4d67602692c21bf5293a75e4b607bb92a11e821e2b859a"}, - {file = "greenlet-3.0.1-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:41bdeeb552d814bcd7fb52172b304898a35818107cc8778b5101423c9017b3de"}, - {file = "greenlet-3.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:6e6061bf1e9565c29002e3c601cf68569c450be7fc3f7336671af7ddb4657166"}, - {file = "greenlet-3.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:fa24255ae3c0ab67e613556375a4341af04a084bd58764731972bcbc8baeba36"}, - {file = "greenlet-3.0.1-cp37-cp37m-win32.whl", hash = "sha256:b489c36d1327868d207002391f662a1d163bdc8daf10ab2e5f6e41b9b96de3b1"}, - {file = "greenlet-3.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:f33f3258aae89da191c6ebaa3bc517c6c4cbc9b9f689e5d8452f7aedbb913fa8"}, - {file = "greenlet-3.0.1-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:d2905ce1df400360463c772b55d8e2518d0e488a87cdea13dd2c71dcb2a1fa16"}, - {file = "greenlet-3.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0a02d259510b3630f330c86557331a3b0e0c79dac3d166e449a39363beaae174"}, - {file = "greenlet-3.0.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:55d62807f1c5a1682075c62436702aaba941daa316e9161e4b6ccebbbf38bda3"}, - {file = "greenlet-3.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3fcc780ae8edbb1d050d920ab44790201f027d59fdbd21362340a85c79066a74"}, - {file = "greenlet-3.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4eddd98afc726f8aee1948858aed9e6feeb1758889dfd869072d4465973f6bfd"}, - {file = "greenlet-3.0.1-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:eabe7090db68c981fca689299c2d116400b553f4b713266b130cfc9e2aa9c5a9"}, - {file = "greenlet-3.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f2f6d303f3dee132b322a14cd8765287b8f86cdc10d2cb6a6fae234ea488888e"}, - {file = "greenlet-3.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d923ff276f1c1f9680d32832f8d6c040fe9306cbfb5d161b0911e9634be9ef0a"}, - {file = "greenlet-3.0.1-cp38-cp38-win32.whl", hash = "sha256:0b6f9f8ca7093fd4433472fd99b5650f8a26dcd8ba410e14094c1e44cd3ceddd"}, - {file = "greenlet-3.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:990066bff27c4fcf3b69382b86f4c99b3652bab2a7e685d968cd4d0cfc6f67c6"}, - {file = "greenlet-3.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:ce85c43ae54845272f6f9cd8320d034d7a946e9773c693b27d620edec825e376"}, - {file = "greenlet-3.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:89ee2e967bd7ff85d84a2de09df10e021c9b38c7d91dead95b406ed6350c6997"}, - {file = "greenlet-3.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:87c8ceb0cf8a5a51b8008b643844b7f4a8264a2c13fcbcd8a8316161725383fe"}, - {file = "greenlet-3.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d6a8c9d4f8692917a3dc7eb25a6fb337bff86909febe2f793ec1928cd97bedfc"}, - {file = "greenlet-3.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9fbc5b8f3dfe24784cee8ce0be3da2d8a79e46a276593db6868382d9c50d97b1"}, - {file = "greenlet-3.0.1-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:85d2b77e7c9382f004b41d9c72c85537fac834fb141b0296942d52bf03fe4a3d"}, - {file = "greenlet-3.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:696d8e7d82398e810f2b3622b24e87906763b6ebfd90e361e88eb85b0e554dc8"}, - {file = "greenlet-3.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:329c5a2e5a0ee942f2992c5e3ff40be03e75f745f48847f118a3cfece7a28546"}, - {file = "greenlet-3.0.1-cp39-cp39-win32.whl", hash = "sha256:cf868e08690cb89360eebc73ba4be7fb461cfbc6168dd88e2fbbe6f31812cd57"}, - {file = "greenlet-3.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:ac4a39d1abae48184d420aa8e5e63efd1b75c8444dd95daa3e03f6c6310e9619"}, - {file = "greenlet-3.0.1.tar.gz", hash = "sha256:816bd9488a94cba78d93e1abb58000e8266fa9cc2aa9ccdd6eb0696acb24005b"}, + {file = "greenlet-3.0.2-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9acd8fd67c248b8537953cb3af8787c18a87c33d4dcf6830e410ee1f95a63fd4"}, + {file = "greenlet-3.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:339c0272a62fac7e602e4e6ec32a64ff9abadc638b72f17f6713556ed011d493"}, + {file = "greenlet-3.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:38878744926cec29b5cc3654ef47f3003f14bfbba7230e3c8492393fe29cc28b"}, + {file = "greenlet-3.0.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b3f0497db77cfd034f829678b28267eeeeaf2fc21b3f5041600f7617139e6773"}, + {file = "greenlet-3.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ed1a8a08de7f68506a38f9a2ddb26bbd1480689e66d788fcd4b5f77e2d9ecfcc"}, + {file = "greenlet-3.0.2-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:89a6f6ddcbef4000cda7e205c4c20d319488ff03db961d72d4e73519d2465309"}, + {file = "greenlet-3.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:c1f647fe5b94b51488b314c82fdda10a8756d650cee8d3cd29f657c6031bdf73"}, + {file = "greenlet-3.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9560c580c896030ff9c311c603aaf2282234643c90d1dec738a1d93e3e53cd51"}, + {file = "greenlet-3.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2e9c5423046eec21f6651268cb674dfba97280701e04ef23d312776377313206"}, + {file = "greenlet-3.0.2-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b1fd25dfc5879a82103b3d9e43fa952e3026c221996ff4d32a9c72052544835d"}, + {file = "greenlet-3.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cecfdc950dd25f25d6582952e58521bca749cf3eeb7a9bad69237024308c8196"}, + {file = "greenlet-3.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:edf7a1daba1f7c54326291a8cde58da86ab115b78c91d502be8744f0aa8e3ffa"}, + {file = "greenlet-3.0.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f4cf532bf3c58a862196b06947b1b5cc55503884f9b63bf18582a75228d9950e"}, + {file = "greenlet-3.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e79fb5a9fb2d0bd3b6573784f5e5adabc0b0566ad3180a028af99523ce8f6138"}, + {file = "greenlet-3.0.2-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:006c1028ac0cfcc4e772980cfe73f5476041c8c91d15d64f52482fc571149d46"}, + {file = "greenlet-3.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:fefd5eb2c0b1adffdf2802ff7df45bfe65988b15f6b972706a0e55d451bffaea"}, + {file = "greenlet-3.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0c0fdb8142742ee68e97c106eb81e7d3e883cc739d9c5f2b28bc38a7bafeb6d1"}, + {file = "greenlet-3.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:8f8d14a0a4e8c670fbce633d8b9a1ee175673a695475acd838e372966845f764"}, + {file = "greenlet-3.0.2-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:654b84c9527182036747938b81938f1d03fb8321377510bc1854a9370418ab66"}, + {file = "greenlet-3.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd5bc4fde0842ff2b9cf33382ad0b4db91c2582db836793d58d174c569637144"}, + {file = "greenlet-3.0.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c27b142a9080bdd5869a2fa7ebf407b3c0b24bd812db925de90e9afe3c417fd6"}, + {file = "greenlet-3.0.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0df7eed98ea23b20e9db64d46eb05671ba33147df9405330695bcd81a73bb0c9"}, + {file = "greenlet-3.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb5d60805057d8948065338be6320d35e26b0a72f45db392eb32b70dd6dc9227"}, + {file = "greenlet-3.0.2-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e0e28f5233d64c693382f66d47c362b72089ebf8ac77df7e12ac705c9fa1163d"}, + {file = "greenlet-3.0.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:3e4bfa752b3688d74ab1186e2159779ff4867644d2b1ebf16db14281f0445377"}, + {file = "greenlet-3.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:c42bb589e6e9f9d8bdd79f02f044dff020d30c1afa6e84c0b56d1ce8a324553c"}, + {file = "greenlet-3.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:b2cedf279ca38ef3f4ed0d013a6a84a7fc3d9495a716b84a5fc5ff448965f251"}, + {file = "greenlet-3.0.2-cp37-cp37m-macosx_11_0_universal2.whl", hash = "sha256:6d65bec56a7bc352bcf11b275b838df618651109074d455a772d3afe25390b7d"}, + {file = "greenlet-3.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0acadbc3f72cb0ee85070e8d36bd2a4673d2abd10731ee73c10222cf2dd4713c"}, + {file = "greenlet-3.0.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:14b5d999aefe9ffd2049ad19079f733c3aaa426190ffecadb1d5feacef8fe397"}, + {file = "greenlet-3.0.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:f27aa32466993c92d326df982c4acccd9530fe354e938d9e9deada563e71ce76"}, + {file = "greenlet-3.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f34a765c5170c0673eb747213a0275ecc749ab3652bdbec324621ed5b2edaef"}, + {file = "greenlet-3.0.2-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:520fcb53a39ef90f5021c77606952dbbc1da75d77114d69b8d7bded4a8e1a813"}, + {file = "greenlet-3.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d1fceb5351ab1601903e714c3028b37f6ea722be6873f46e349a960156c05650"}, + {file = "greenlet-3.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:7363756cc439a503505b67983237d1cc19139b66488263eb19f5719a32597836"}, + {file = "greenlet-3.0.2-cp37-cp37m-win32.whl", hash = "sha256:d5547b462b8099b84746461e882a3eb8a6e3f80be46cb6afb8524eeb191d1a30"}, + {file = "greenlet-3.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:950e21562818f9c771989b5b65f990e76f4ac27af66e1bb34634ae67886ede2a"}, + {file = "greenlet-3.0.2-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:d64643317e76b4b41fdba659e7eca29634e5739b8bc394eda3a9127f697ed4b0"}, + {file = "greenlet-3.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f9ea7c2c9795549653b6f7569f6bc75d2c7d1f6b2854eb8ce0bc6ec3cb2dd88"}, + {file = "greenlet-3.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:db4233358d3438369051a2f290f1311a360d25c49f255a6c5d10b5bcb3aa2b49"}, + {file = "greenlet-3.0.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ed9bf77b41798e8417657245b9f3649314218a4a17aefb02bb3992862df32495"}, + {file = "greenlet-3.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d4d0df07a38e41a10dfb62c6fc75ede196572b580f48ee49b9282c65639f3965"}, + {file = "greenlet-3.0.2-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:10d247260db20887ae8857c0cbc750b9170f0b067dd7d38fb68a3f2334393bd3"}, + {file = "greenlet-3.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:a37ae53cca36823597fd5f65341b6f7bac2dd69ecd6ca01334bb795460ab150b"}, + {file = "greenlet-3.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:80d068e4b6e2499847d916ef64176811ead6bf210a610859220d537d935ec6fd"}, + {file = "greenlet-3.0.2-cp38-cp38-win32.whl", hash = "sha256:b1405614692ac986490d10d3e1a05e9734f473750d4bee3cf7d1286ef7af7da6"}, + {file = "greenlet-3.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:8756a94ed8f293450b0e91119eca2a36332deba69feb2f9ca410d35e74eae1e4"}, + {file = "greenlet-3.0.2-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:2c93cd03acb1499ee4de675e1a4ed8eaaa7227f7949dc55b37182047b006a7aa"}, + {file = "greenlet-3.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1dac09e3c0b78265d2e6d3cbac2d7c48bd1aa4b04a8ffeda3adde9f1688df2c3"}, + {file = "greenlet-3.0.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2ee59c4627c8c4bb3e15949fbcd499abd6b7f4ad9e0bfcb62c65c5e2cabe0ec4"}, + {file = "greenlet-3.0.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18fe39d70d482b22f0014e84947c5aaa7211fb8e13dc4cc1c43ed2aa1db06d9a"}, + {file = "greenlet-3.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e84bef3cfb6b6bfe258c98c519811c240dbc5b33a523a14933a252e486797c90"}, + {file = "greenlet-3.0.2-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:aecea0442975741e7d69daff9b13c83caff8c13eeb17485afa65f6360a045765"}, + {file = "greenlet-3.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:f260e6c2337871a52161824058923df2bbddb38bc11a5cbe71f3474d877c5bd9"}, + {file = "greenlet-3.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fc14dd9554f88c9c1fe04771589ae24db76cd56c8f1104e4381b383d6b71aff8"}, + {file = "greenlet-3.0.2-cp39-cp39-win32.whl", hash = "sha256:bfcecc984d60b20ffe30173b03bfe9ba6cb671b0be1e95c3e2056d4fe7006590"}, + {file = "greenlet-3.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:c235131bf59d2546bb3ebaa8d436126267392f2e51b85ff45ac60f3a26549af0"}, + {file = "greenlet-3.0.2.tar.gz", hash = "sha256:1c1129bc47266d83444c85a8e990ae22688cf05fb20d7951fd2866007c2ba9bc"}, ] [package.extras] @@ -1666,21 +1666,15 @@ files = [ [[package]] name = "isort" -version = "5.12.0" +version = "5.13.1" description = "A Python utility / library to sort Python imports." optional = false python-versions = ">=3.8.0" files = [ - {file = "isort-5.12.0-py3-none-any.whl", hash = "sha256:f84c2818376e66cf843d497486ea8fed8700b340f308f076c6fb1229dff318b6"}, - {file = "isort-5.12.0.tar.gz", hash = "sha256:8bef7dde241278824a6d83f44a544709b065191b95b6e50894bdc722fcba0504"}, + {file = "isort-5.13.1-py3-none-any.whl", hash = "sha256:56a51732c25f94ca96f6721be206dd96a95f42950502eb26c1015d333bc6edb7"}, + {file = "isort-5.13.1.tar.gz", hash = "sha256:aaed790b463e8703fb1eddb831dfa8e8616bacde2c083bd557ef73c8189b7263"}, ] -[package.extras] -colors = ["colorama (>=0.4.3)"] -pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"] -plugins = ["setuptools"] -requirements-deprecated-finder = ["pip-api", "pipreqs"] - [[package]] name = "jaraco-classes" version = "3.3.0" @@ -2401,13 +2395,13 @@ test = ["coverage[toml]", "pymdown-extensions", "pytest", "pytest-cov"] [[package]] name = "mkdocs-material" -version = "9.5.0" +version = "9.5.2" description = "Documentation that simply works" optional = false python-versions = ">=3.8" files = [ - {file = "mkdocs_material-9.5.0-py3-none-any.whl", hash = "sha256:47323c9943756ced14487d0607caa283e350e8fd4c724b224bd69ccb0b0731f2"}, - {file = "mkdocs_material-9.5.0.tar.gz", hash = "sha256:f8f9ccb526bd8a8ac2e5ec2e64e1a167cfb53cf6a84c8924dbb607f0a70e74ae"}, + {file = "mkdocs_material-9.5.2-py3-none-any.whl", hash = "sha256:6ed0fbf4682491766f0ec1acc955db6901c2fd424c7ab343964ef51b819741f5"}, + {file = "mkdocs_material-9.5.2.tar.gz", hash = "sha256:ca8b9cd2b3be53e858e5a1a45ac9668bd78d95d77a30288bb5ebc1a31db6184c"}, ] [package.dependencies] @@ -3016,17 +3010,17 @@ files = [ [[package]] name = "openai" -version = "1.3.7" +version = "1.3.8" description = "The official Python library for the openai API" optional = false python-versions = ">=3.7.1" files = [ - {file = "openai-1.3.7-py3-none-any.whl", hash = "sha256:e5c51367a910297e4d1cd33d2298fb87d7edf681edbe012873925ac16f95bee0"}, - {file = "openai-1.3.7.tar.gz", hash = "sha256:18074a0f51f9b49d1ae268c7abc36f7f33212a0c0d08ce11b7053ab2d17798de"}, + {file = "openai-1.3.8-py3-none-any.whl", hash = "sha256:ac5a17352b96db862390d2e6f51de9f7eb32e733f412467b2f160fbd3d0f2609"}, + {file = "openai-1.3.8.tar.gz", hash = "sha256:54963ff247abe185aad6ee443820e48ad9f87eb4de970acb2514bc113ced748c"}, ] [package.dependencies] -anyio = ">=3.5.0,<4" +anyio = ">=3.5.0,<5" distro = ">=1.7.0,<2" httpx = ">=0.23.0,<1" pydantic = ">=1.9.0,<3" @@ -3154,8 +3148,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -3212,13 +3206,13 @@ testing = ["docopt", "pytest (<6.0.0)"] [[package]] name = "pathspec" -version = "0.11.2" +version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "pathspec-0.11.2-py3-none-any.whl", hash = "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20"}, - {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, ] [[package]] @@ -3498,13 +3492,13 @@ pycryptodome = ">=3.10.1" [[package]] name = "prompt-toolkit" -version = "3.0.41" +version = "3.0.42" description = "Library for building powerful interactive command lines in Python" optional = false python-versions = ">=3.7.0" files = [ - {file = "prompt_toolkit-3.0.41-py3-none-any.whl", hash = "sha256:f36fe301fafb7470e86aaf90f036eef600a3210be4decf461a5b1ca8403d3cb2"}, - {file = "prompt_toolkit-3.0.41.tar.gz", hash = "sha256:941367d97fc815548822aa26c2a269fdc4eb21e9ec05fc5d447cf09bad5d75f0"}, + {file = "prompt_toolkit-3.0.42-py3-none-any.whl", hash = "sha256:3b50b5fc50660dc8e39dfe464b170959ad82ff185ffa53bfd3be02222e7156a1"}, + {file = "prompt_toolkit-3.0.42.tar.gz", hash = "sha256:bfbf7d6ea9744e4ec94c9a69539e8106c77a2a607d728ded87c9182a4aec39be"}, ] [package.dependencies] @@ -3829,24 +3823,24 @@ extra = ["pygments (>=2.12)"] [[package]] name = "pypdfium2" -version = "4.24.0" +version = "4.25.0" description = "Python bindings to PDFium" optional = false python-versions = ">= 3.6" files = [ - {file = "pypdfium2-4.24.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:94b722c4dddbd858d62fe4df3192651f9376f1c99e7c2bc74d7d8c8d06362bf3"}, - {file = "pypdfium2-4.24.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:c2891aa6059acf9bdabccb7aa193f111ebf96fabae3fb968f04ec925d710ec95"}, - {file = "pypdfium2-4.24.0-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:147d101686af8828fbaeb3ac3fd82114f0900d58a24e80eff96496fd89fd9d2d"}, - {file = "pypdfium2-4.24.0-py3-none-manylinux_2_17_armv7l.whl", hash = "sha256:60c7d9c442aff40d30dbf044ffb67cdc5eb56acca59ac640bc3adad77fc4d781"}, - {file = "pypdfium2-4.24.0-py3-none-manylinux_2_17_i686.whl", hash = "sha256:025553c8b3633b32e2ef0e9ec9ee07be4a4fda76519889607ad3283090eef7f1"}, - {file = "pypdfium2-4.24.0-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:3b26ad59ebef92edfcb44400838edce2e299a9709fe472742a4800251b30e5c9"}, - {file = "pypdfium2-4.24.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:4034e6b4bde7cb6d281898c43ccb9a5522e25edb1e24689bf89fc7eb2a0c9a15"}, - {file = "pypdfium2-4.24.0-py3-none-musllinux_1_1_i686.whl", hash = "sha256:a1776dde55b55d81e18026cf746274c1e2959bc8ed2f502a997401e1f0e7c3c1"}, - {file = "pypdfium2-4.24.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:39a1e1cc02bc39233f742b8cdf60a81e5b4868bfee28ec79457e37e3d41304e6"}, - {file = "pypdfium2-4.24.0-py3-none-win32.whl", hash = "sha256:7556801f2b42c91590e3f862034ab61e30e732b09e1487b0cf1a3c5250cb29d4"}, - {file = "pypdfium2-4.24.0-py3-none-win_amd64.whl", hash = "sha256:fa65834fbc6540114ceaebc5e9ca90c5455b0ebedaaaf6c2c8351c851ada366b"}, - {file = "pypdfium2-4.24.0-py3-none-win_arm64.whl", hash = "sha256:9333304e289fa727fbeae6dab793a9bacb68375184e14ad3d38a65d9a7490be1"}, - {file = "pypdfium2-4.24.0.tar.gz", hash = "sha256:62706c06bc5be39aa7a2531af802420429b6c4c47498eebd2521af7e988d0848"}, + {file = "pypdfium2-4.25.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:25075d85834bf70a2244ce564063ee9aa2c738a019c09aeffa61920163892110"}, + {file = "pypdfium2-4.25.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:ab46ac5e257b0610ca2bed7b5baf588b1417abe5bc36339ffdc651620dfe02f8"}, + {file = "pypdfium2-4.25.0-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:7b5131374574d4f602346d1ef489362cc7fbd7a1920214e14efd5114c1915bf5"}, + {file = "pypdfium2-4.25.0-py3-none-manylinux_2_17_armv7l.whl", hash = "sha256:20cb1d9fbd78595f0d0750a232f8204caa2f2aec34a1dde80ec5184f1efcf90d"}, + {file = "pypdfium2-4.25.0-py3-none-manylinux_2_17_i686.whl", hash = "sha256:48ab21bed55bcb2cbce5c61363fc0f5481b26f5eb34b484fbea126c0d86f3697"}, + {file = "pypdfium2-4.25.0-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:766730b0422347770189de5d3c7ff0c85620be12678d813fcce1122900b31c40"}, + {file = "pypdfium2-4.25.0-py3-none-musllinux_1_1_aarch64.whl", hash = "sha256:485b404bd059a80a1bb0646647e2f0493f9240ca2c2672059d876b91c2e7e9f9"}, + {file = "pypdfium2-4.25.0-py3-none-musllinux_1_1_i686.whl", hash = "sha256:f49234b882c5c3fd1936fa20db665d6667d45bcbefc8120565c600aa7287bda0"}, + {file = "pypdfium2-4.25.0-py3-none-musllinux_1_1_x86_64.whl", hash = "sha256:b43a0206c2ac5dc812b6bdff2e74023a080f06ca847dd7905063f33ca385c655"}, + {file = "pypdfium2-4.25.0-py3-none-win32.whl", hash = "sha256:23e693cceb0154a609838bf827a139a475eb532297d2c2b6cd4033b9bac73dad"}, + {file = "pypdfium2-4.25.0-py3-none-win_amd64.whl", hash = "sha256:43f7612bc802518d67dacec11d1d0fb447c7371ce70cb2328f70ef71e9fe0582"}, + {file = "pypdfium2-4.25.0-py3-none-win_arm64.whl", hash = "sha256:d05155f6a50b8025c0fe0a11a527d4f1716c09f06fa4e4d46f3798fd040e8b95"}, + {file = "pypdfium2-4.25.0.tar.gz", hash = "sha256:1691bfa3e65bc84fd8b1d26a6d68af90f9662771a647dd6c4e031d81c4a72037"}, ] [[package]] @@ -4706,13 +4700,13 @@ files = [ [[package]] name = "sacrebleu" -version = "2.3.3" +version = "2.4.0" description = "Hassle-free computation of shareable, comparable, and reproducible BLEU, chrF, and TER scores" optional = true python-versions = ">=3.6" files = [ - {file = "sacrebleu-2.3.3-py3-none-any.whl", hash = "sha256:215422e29c615823281d7aa1c85c2955b9a810eee7c59909c605e8e62ee63478"}, - {file = "sacrebleu-2.3.3.tar.gz", hash = "sha256:eff9cfd2c91041f57f6681c646c9b230ae332e442640762a556605d6ef67465b"}, + {file = "sacrebleu-2.4.0-py3-none-any.whl", hash = "sha256:fc7c34464a56d691bf5e37c4b5292142d2273b02516ac61e264cd19035fff981"}, + {file = "sacrebleu-2.4.0.tar.gz", hash = "sha256:d9e918147dc0777b2e159bff3246b8eb22d76f3b4ee3e6c6cbda05dc25dbb9c0"}, ] [package.dependencies] @@ -4724,7 +4718,6 @@ regex = "*" tabulate = ">=0.8.9" [package.extras] -dev = ["lxml-stubs", "mypy", "pytest", "types-tabulate", "wheel"] ja = ["ipadic (>=1.0,<2.0)", "mecab-python3 (>=1.0.5,<=1.0.6)"] ko = ["mecab-ko (>=1.0.0,<=1.0.1)", "mecab-ko-dic (>=1.0,<2.0)"] @@ -6144,13 +6137,13 @@ test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6. [[package]] name = "typing-extensions" -version = "4.8.0" +version = "4.9.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.8.0-py3-none-any.whl", hash = "sha256:8f92fc8806f9a6b641eaa5318da32b44d401efaac0f6678c9bc448ba3605faa0"}, - {file = "typing_extensions-4.8.0.tar.gz", hash = "sha256:df8e4339e9cb77357558cbdbceca33c303714cf861d1eef15e1070055ae8b7ef"}, + {file = "typing_extensions-4.9.0-py3-none-any.whl", hash = "sha256:af72aea155e91adfc61c3ae9e0e342dbc0cba726d6cba4b6c72c1f34e47291cd"}, + {file = "typing_extensions-4.9.0.tar.gz", hash = "sha256:23478f88c37f27d76ac8aee6c905017a143b0b1b886c3c9f66bc2fd94f9f5783"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 3abc87237..ee21cf4b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "guardrails-ai" -version = "0.3.0" +version = "0.3.1" description = "Adding guardrails to large language models." authors = ["Guardrails AI "] license = "Apache License 2.0" From 1b14f1088cfc63a7e3a852f017717758ed20b8d8 Mon Sep 17 00:00:00 2001 From: zsimjee Date: Tue, 12 Dec 2023 16:03:05 -0800 Subject: [PATCH 11/20] try to release without a fixed commit v of comet --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index ee21cf4b1..a355a3edb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ presidio_anonymizer = {version = "^2.2.33", optional = true} spacy-transformers = {version = "^1.3.3", optional = true} anthropic = {version = "^0.7.2", optional = true} torch = {version = "^2.1.1", optional = true} -unbabel-comet = {git = "https://github.com/Unbabel/COMET.git", rev="45cb572", optional = true} +unbabel-comet = {git = "https://github.com/Unbabel/COMET.git", optional = true} huggingface_hub = {version = "^0.16.4", optional = true} From 27da7bd7410246a76c43b3573ee249328b078b41 Mon Sep 17 00:00:00 2001 From: zsimjee Date: Tue, 12 Dec 2023 16:09:24 -0800 Subject: [PATCH 12/20] Revert "try to release without a fixed commit v of comet" This reverts commit 29e708ba2c0b78e6876bb09615e7a3eca69b7954. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index a355a3edb..ee21cf4b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ presidio_anonymizer = {version = "^2.2.33", optional = true} spacy-transformers = {version = "^1.3.3", optional = true} anthropic = {version = "^0.7.2", optional = true} torch = {version = "^2.1.1", optional = true} -unbabel-comet = {git = "https://github.com/Unbabel/COMET.git", optional = true} +unbabel-comet = {git = "https://github.com/Unbabel/COMET.git", rev="45cb572", optional = true} huggingface_hub = {version = "^0.16.4", optional = true} From d6dc4c2e540e96c13358e5c650ecfec8bbbc6e55 Mon Sep 17 00:00:00 2001 From: zsimjee Date: Tue, 12 Dec 2023 16:52:32 -0800 Subject: [PATCH 13/20] use guardrails-dist version of comet --- poetry.lock | 64 +++++++++++++++++++++++--------------------------- pyproject.toml | 4 ++-- 2 files changed, 32 insertions(+), 36 deletions(-) diff --git a/poetry.lock b/poetry.lock index db9da678e..979c7f66e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -1420,6 +1420,32 @@ files = [ [package.dependencies] colorama = ">=0.4" +[[package]] +name = "guardrails-ai-unbabel-comet" +version = "2.2.1" +description = "High-quality Machine Translation Evaluation" +optional = true +python-versions = ">=3.8.0,<4.0.0" +files = [ + {file = "guardrails_ai_unbabel_comet-2.2.1-py3-none-any.whl", hash = "sha256:4969432a868fc57f6cdfa3acb9b5c0eb9e5dc73352d88a8ee07f8ebbd7931760"}, + {file = "guardrails_ai_unbabel_comet-2.2.1.tar.gz", hash = "sha256:0d79dbb49c01bfb7e63e4115a527a4bb9abc34e9fbc908bd3c80d7f19ddad243"}, +] + +[package.dependencies] +entmax = ">=1.1,<2.0" +huggingface-hub = ">=0.16.0,<0.17.0" +jsonargparse = "3.13.1" +numpy = ">=1.20.0,<2.0.0" +pandas = ">=1.4.1" +protobuf = ">=4.24.4,<5.0.0" +pytorch-lightning = ">=2.0.0,<3.0.0" +sacrebleu = ">=2.0.0,<3.0.0" +scipy = ">=1.5.4,<2.0.0" +sentencepiece = ">=0.1.96,<0.2.0" +torch = ">=1.6.0" +torchmetrics = ">=0.10.2,<0.11.0" +transformers = ">=4.17,<5.0" + [[package]] name = "h11" version = "0.14.0" @@ -3148,8 +3174,8 @@ files = [ [package.dependencies] numpy = [ {version = ">=1.20.3", markers = "python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, ] python-dateutil = ">=2.8.2" pytz = ">=2020.1" @@ -6157,36 +6183,6 @@ files = [ {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, ] -[[package]] -name = "unbabel-comet" -version = "2.2.0" -description = "High-quality Machine Translation Evaluation" -optional = true -python-versions = "^3.8.0" -files = [] -develop = false - -[package.dependencies] -entmax = "^1.1" -huggingface-hub = "^0.16.0" -jsonargparse = "3.13.1" -numpy = "^1.20.0" -pandas = ">=1.4.1" -protobuf = "^4.24.4" -pytorch-lightning = "^2.0.0" -sacrebleu = "^2.0.0" -scipy = "^1.5.4" -sentencepiece = "^0.1.96" -torch = ">=1.6.0" -torchmetrics = "^0.10.2" -transformers = "^4.17" - -[package.source] -type = "git" -url = "https://github.com/Unbabel/COMET.git" -reference = "45cb572" -resolved_reference = "45cb572516398b6994f112ed8ee7058dc5fb84ef" - [[package]] name = "untokenize" version = "0.1.1" @@ -6632,7 +6628,7 @@ testing = ["coverage (>=5.0.3)", "zope.event", "zope.testing"] anthropic = ["anthropic"] competitor-check = ["spacy-transformers"] detect-secrets = ["detect-secrets"] -high-quality-translation = ["huggingface_hub", "unbabel-comet"] +high-quality-translation = ["guardrails-ai-unbabel-comet", "huggingface_hub"] manifest = ["manifest-ml"] pii = ["presidio_analyzer", "presidio_anonymizer"] profanity = ["alt-profanity-check"] @@ -6644,4 +6640,4 @@ vectordb = ["faiss-cpu", "numpy", "tiktoken"] [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "3959055900713c27ac972c653de87772805b5ff5325362922eddad03700d8f0b" +content-hash = "6915b83d33c66b3798ca0cd4417cd179fa9ed04844f0656490abfa7ee2b61b51" diff --git a/pyproject.toml b/pyproject.toml index ee21cf4b1..b4d934da8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -45,7 +45,7 @@ presidio_anonymizer = {version = "^2.2.33", optional = true} spacy-transformers = {version = "^1.3.3", optional = true} anthropic = {version = "^0.7.2", optional = true} torch = {version = "^2.1.1", optional = true} -unbabel-comet = {git = "https://github.com/Unbabel/COMET.git", rev="45cb572", optional = true} +guardrails-ai-unbabel-comet = {version = "^2.2.1", optional = true} huggingface_hub = {version = "^0.16.4", optional = true} @@ -56,7 +56,7 @@ vectordb = ["faiss-cpu", "tiktoken", "numpy"] profanity = ["alt-profanity-check"] detect-secrets = ["detect-secrets"] manifest = ["manifest-ml"] -high_quality_translation = ["unbabel-comet", "huggingface_hub"] +high_quality_translation = ["guardrails-ai-unbabel-comet", "huggingface_hub"] toxic_language = ["transformers", "torch"] pii = ["presidio_analyzer", "presidio_anonymizer"] competitor-check = ["spacy-transformers"] From 4088893a72e0ab2a17c45f99402285a164a23b95 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Wed, 13 Dec 2023 14:21:43 -0600 Subject: [PATCH 14/20] better fix logic for TwoWord validator --- docs/examples/extracting_entities.ipynb | 156 +++++++++++------- guardrails/utils/reask_utils.py | 17 +- guardrails/validators/two_words.py | 15 +- poetry.lock | 19 ++- pyproject.toml | 1 + .../entity_extraction/validated_output_fix.py | 2 +- .../validated_output_reask_1.py | 2 +- .../validated_output_reask_2.py | 2 +- .../validated_output_skeleton_reask_2.py | 18 +- tests/unit_tests/test_validators.py | 2 +- tests/unit_tests/validators/test_two_words.py | 52 ++++++ 11 files changed, 207 insertions(+), 79 deletions(-) create mode 100644 tests/unit_tests/validators/test_two_words.py diff --git a/docs/examples/extracting_entities.ipynb b/docs/examples/extracting_entities.ipynb index a15a0ca90..69366243b 100644 --- a/docs/examples/extracting_entities.ipynb +++ b/docs/examples/extracting_entities.ipynb @@ -347,16 +347,16 @@ "text/html": [ "
{\n",
        "    'fees': [\n",
-       "        {'index': 1, 'name': 'annual_membership', 'explanation': 'Annual Membership Fee', 'value': 0.0},\n",
+       "        {'index': 1, 'name': 'annual membership', 'explanation': 'Annual Membership Fee', 'value': 0.0},\n",
        "        {\n",
        "            'index': 2,\n",
-       "            'name': 'my_chase_plan',\n",
+       "            'name': 'my chase',\n",
        "            'explanation': 'My Chase Plan Fee (fixed finance charge)',\n",
        "            'value': 1.72\n",
        "        },\n",
        "        {\n",
        "            'index': 3,\n",
-       "            'name': 'balance_transfers',\n",
+       "            'name': 'balance transfers',\n",
        "            'explanation': 'Balance Transfers Intro fee of either $5 or 3% of the amount of each transfer, \n",
        "whichever is greater, on transfers made within 60 days of account opening. After that: Either $5 or 5% of the \n",
        "amount of each transfer, whichever is greater.',\n",
@@ -364,20 +364,20 @@
        "        },\n",
        "        {\n",
        "            'index': 4,\n",
-       "            'name': 'cash_advances',\n",
+       "            'name': 'cash advances',\n",
        "            'explanation': 'Either $10 or 5% of the amount of each transaction, whichever is greater.',\n",
        "            'value': 5.0\n",
        "        },\n",
        "        {\n",
        "            'index': 5,\n",
-       "            'name': 'foreign_transactions',\n",
+       "            'name': 'foreign transactions',\n",
        "            'explanation': 'Foreign Transactions 3% of the amount of each transaction in U.S. dollars.',\n",
        "            'value': 3.0\n",
        "        },\n",
-       "        {'index': 6, 'name': 'late_payment', 'explanation': 'Late Payment Up to $40.', 'value': 0.0},\n",
-       "        {'index': 7, 'name': 'over_the_credit_limit', 'explanation': 'Over-the-Credit-Limit None', 'value': 0.0},\n",
-       "        {'index': 8, 'name': 'return_payment', 'explanation': 'Return Payment Up to $40.', 'value': 0.0},\n",
-       "        {'index': 9, 'name': 'return_check', 'explanation': 'Return Check None', 'value': 0.0}\n",
+       "        {'index': 6, 'name': 'late payment', 'explanation': 'Late Payment Up to $40.', 'value': 0.0},\n",
+       "        {'index': 7, 'name': 'over the', 'explanation': 'Over-the-Credit-Limit None', 'value': 0.0},\n",
+       "        {'index': 8, 'name': 'return payment', 'explanation': 'Return Payment Up to $40.', 'value': 0.0},\n",
+       "        {'index': 9, 'name': 'return check', 'explanation': 'Return Check None', 'value': 0.0}\n",
        "    ],\n",
        "    'interest_rates': {\n",
        "        'purchase': {'apr': 0, 'after_apr': 19.49},\n",
@@ -392,16 +392,16 @@
       "text/plain": [
        "\u001b[1m{\u001b[0m\n",
        "    \u001b[32m'fees'\u001b[0m: \u001b[1m[\u001b[0m\n",
-       "        \u001b[1m{\u001b[0m\u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'name'\u001b[0m: \u001b[32m'annual_membership'\u001b[0m, \u001b[32m'explanation'\u001b[0m: \u001b[32m'Annual Membership Fee'\u001b[0m, \u001b[32m'value'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[1m{\u001b[0m\u001b[32m'index'\u001b[0m: \u001b[1;36m1\u001b[0m, \u001b[32m'name'\u001b[0m: \u001b[32m'annual membership'\u001b[0m, \u001b[32m'explanation'\u001b[0m: \u001b[32m'Annual Membership Fee'\u001b[0m, \u001b[32m'value'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m,\n",
        "        \u001b[1m{\u001b[0m\n",
        "            \u001b[32m'index'\u001b[0m: \u001b[1;36m2\u001b[0m,\n",
-       "            \u001b[32m'name'\u001b[0m: \u001b[32m'my_chase_plan'\u001b[0m,\n",
+       "            \u001b[32m'name'\u001b[0m: \u001b[32m'my chase'\u001b[0m,\n",
        "            \u001b[32m'explanation'\u001b[0m: \u001b[32m'My Chase Plan Fee \u001b[0m\u001b[32m(\u001b[0m\u001b[32mfixed finance charge\u001b[0m\u001b[32m)\u001b[0m\u001b[32m'\u001b[0m,\n",
        "            \u001b[32m'value'\u001b[0m: \u001b[1;36m1.72\u001b[0m\n",
        "        \u001b[1m}\u001b[0m,\n",
        "        \u001b[1m{\u001b[0m\n",
        "            \u001b[32m'index'\u001b[0m: \u001b[1;36m3\u001b[0m,\n",
-       "            \u001b[32m'name'\u001b[0m: \u001b[32m'balance_transfers'\u001b[0m,\n",
+       "            \u001b[32m'name'\u001b[0m: \u001b[32m'balance transfers'\u001b[0m,\n",
        "            \u001b[32m'explanation'\u001b[0m: \u001b[32m'Balance Transfers Intro fee of either $5 or 3% of the amount of each transfer, \u001b[0m\n",
        "\u001b[32mwhichever is greater, on transfers made within 60 days of account opening. After that: Either $5 or 5% of the \u001b[0m\n",
        "\u001b[32mamount of each transfer, whichever is greater.'\u001b[0m,\n",
@@ -409,20 +409,20 @@
        "        \u001b[1m}\u001b[0m,\n",
        "        \u001b[1m{\u001b[0m\n",
        "            \u001b[32m'index'\u001b[0m: \u001b[1;36m4\u001b[0m,\n",
-       "            \u001b[32m'name'\u001b[0m: \u001b[32m'cash_advances'\u001b[0m,\n",
+       "            \u001b[32m'name'\u001b[0m: \u001b[32m'cash advances'\u001b[0m,\n",
        "            \u001b[32m'explanation'\u001b[0m: \u001b[32m'Either $10 or 5% of the amount of each transaction, whichever is greater.'\u001b[0m,\n",
        "            \u001b[32m'value'\u001b[0m: \u001b[1;36m5.0\u001b[0m\n",
        "        \u001b[1m}\u001b[0m,\n",
        "        \u001b[1m{\u001b[0m\n",
        "            \u001b[32m'index'\u001b[0m: \u001b[1;36m5\u001b[0m,\n",
-       "            \u001b[32m'name'\u001b[0m: \u001b[32m'foreign_transactions'\u001b[0m,\n",
+       "            \u001b[32m'name'\u001b[0m: \u001b[32m'foreign transactions'\u001b[0m,\n",
        "            \u001b[32m'explanation'\u001b[0m: \u001b[32m'Foreign Transactions 3% of the amount of each transaction in U.S. dollars.'\u001b[0m,\n",
        "            \u001b[32m'value'\u001b[0m: \u001b[1;36m3.0\u001b[0m\n",
        "        \u001b[1m}\u001b[0m,\n",
-       "        \u001b[1m{\u001b[0m\u001b[32m'index'\u001b[0m: \u001b[1;36m6\u001b[0m, \u001b[32m'name'\u001b[0m: \u001b[32m'late_payment'\u001b[0m, \u001b[32m'explanation'\u001b[0m: \u001b[32m'Late Payment Up to $40.'\u001b[0m, \u001b[32m'value'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[1m{\u001b[0m\u001b[32m'index'\u001b[0m: \u001b[1;36m7\u001b[0m, \u001b[32m'name'\u001b[0m: \u001b[32m'over_the_credit_limit'\u001b[0m, \u001b[32m'explanation'\u001b[0m: \u001b[32m'Over-the-Credit-Limit None'\u001b[0m, \u001b[32m'value'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[1m{\u001b[0m\u001b[32m'index'\u001b[0m: \u001b[1;36m8\u001b[0m, \u001b[32m'name'\u001b[0m: \u001b[32m'return_payment'\u001b[0m, \u001b[32m'explanation'\u001b[0m: \u001b[32m'Return Payment Up to $40.'\u001b[0m, \u001b[32m'value'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m,\n",
-       "        \u001b[1m{\u001b[0m\u001b[32m'index'\u001b[0m: \u001b[1;36m9\u001b[0m, \u001b[32m'name'\u001b[0m: \u001b[32m'return_check'\u001b[0m, \u001b[32m'explanation'\u001b[0m: \u001b[32m'Return Check None'\u001b[0m, \u001b[32m'value'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\n",
+       "        \u001b[1m{\u001b[0m\u001b[32m'index'\u001b[0m: \u001b[1;36m6\u001b[0m, \u001b[32m'name'\u001b[0m: \u001b[32m'late payment'\u001b[0m, \u001b[32m'explanation'\u001b[0m: \u001b[32m'Late Payment Up to $40.'\u001b[0m, \u001b[32m'value'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[1m{\u001b[0m\u001b[32m'index'\u001b[0m: \u001b[1;36m7\u001b[0m, \u001b[32m'name'\u001b[0m: \u001b[32m'over the'\u001b[0m, \u001b[32m'explanation'\u001b[0m: \u001b[32m'Over-the-Credit-Limit None'\u001b[0m, \u001b[32m'value'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[1m{\u001b[0m\u001b[32m'index'\u001b[0m: \u001b[1;36m8\u001b[0m, \u001b[32m'name'\u001b[0m: \u001b[32m'return payment'\u001b[0m, \u001b[32m'explanation'\u001b[0m: \u001b[32m'Return Payment Up to $40.'\u001b[0m, \u001b[32m'value'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m,\n",
+       "        \u001b[1m{\u001b[0m\u001b[32m'index'\u001b[0m: \u001b[1;36m9\u001b[0m, \u001b[32m'name'\u001b[0m: \u001b[32m'return check'\u001b[0m, \u001b[32m'explanation'\u001b[0m: \u001b[32m'Return Check None'\u001b[0m, \u001b[32m'value'\u001b[0m: \u001b[1;36m0.0\u001b[0m\u001b[1m}\u001b[0m\n",
        "    \u001b[1m]\u001b[0m,\n",
        "    \u001b[32m'interest_rates'\u001b[0m: \u001b[1m{\u001b[0m\n",
        "        \u001b[32m'purchase'\u001b[0m: \u001b[1m{\u001b[0m\u001b[32m'apr'\u001b[0m: \u001b[1;36m0\u001b[0m, \u001b[32m'after_apr'\u001b[0m: \u001b[1;36m19.49\u001b[0m\u001b[1m}\u001b[0m,\n",
@@ -757,7 +757,7 @@
        "│   │ │                         outcome='fail',                                                                 │ │\n",
        "│   │ │                         metadata=None,                                                                  │ │\n",
        "│   │ │                         error_message='must be exactly two words',                                      │ │\n",
-       "│   │ │                         fix_value='over-the-credit-limit'                                               │ │\n",
+       "│   │ │                         fix_value='over the'                                                            │ │\n",
        "│   │ │                     )                                                                                   │ │\n",
        "│   │ │                 ],                                                                                      │ │\n",
        "│   │ │                 path=['fees', 6, 'name']                                                                │ │\n",
@@ -922,19 +922,19 @@
        "    │ │   \"fees\": [                                                                                             │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 1,                                                                                       │ │\n",
-       "    │ │       \"name\": \"annual_membership\",                                                                      │ │\n",
+       "    │ │       \"name\": \"annual membership\",                                                                      │ │\n",
        "    │ │       \"explanation\": \"Annual Membership Fee\",                                                           │ │\n",
        "    │ │       \"value\": 0.0                                                                                      │ │\n",
        "    │ │     },                                                                                                  │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 2,                                                                                       │ │\n",
-       "    │ │       \"name\": \"my_chase_plan\",                                                                          │ │\n",
+       "    │ │       \"name\": \"my chase plan\",                                                                          │ │\n",
        "    │ │       \"explanation\": \"My Chase Plan Fee (fixed finance charge)\",                                        │ │\n",
        "    │ │       \"value\": 1.72                                                                                     │ │\n",
        "    │ │     },                                                                                                  │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 3,                                                                                       │ │\n",
-       "    │ │       \"name\": \"balance_transfers\",                                                                      │ │\n",
+       "    │ │       \"name\": \"balance transfers\",                                                                      │ │\n",
        "    │ │       \"explanation\": \"Balance Transfers Intro fee of either $5 or 3% of the amount of each transfer,    │ │\n",
        "    │ │ whichever is greater, on transfers made within 60 days of account opening. After that: Either $5 or 5%  │ │\n",
        "    │ │ of the amount of each transfer, whichever is greater.\",                                                 │ │\n",
@@ -942,37 +942,37 @@
        "    │ │     },                                                                                                  │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 4,                                                                                       │ │\n",
-       "    │ │       \"name\": \"cash_advances\",                                                                          │ │\n",
+       "    │ │       \"name\": \"cash advances\",                                                                          │ │\n",
        "    │ │       \"explanation\": \"Either $10 or 5% of the amount of each transaction, whichever is greater.\",       │ │\n",
        "    │ │       \"value\": 5.0                                                                                      │ │\n",
        "    │ │     },                                                                                                  │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 5,                                                                                       │ │\n",
-       "    │ │       \"name\": \"foreign_transactions\",                                                                   │ │\n",
+       "    │ │       \"name\": \"foreign transactions\",                                                                   │ │\n",
        "    │ │       \"explanation\": \"Foreign Transactions 3% of the amount of each transaction in U.S. dollars.\",      │ │\n",
        "    │ │       \"value\": 3.0                                                                                      │ │\n",
        "    │ │     },                                                                                                  │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 6,                                                                                       │ │\n",
-       "    │ │       \"name\": \"late_payment\",                                                                           │ │\n",
+       "    │ │       \"name\": \"late payment\",                                                                           │ │\n",
        "    │ │       \"explanation\": \"Late Payment Up to $40.\",                                                         │ │\n",
        "    │ │       \"value\": 0.0                                                                                      │ │\n",
        "    │ │     },                                                                                                  │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 7,                                                                                       │ │\n",
-       "    │ │       \"name\": \"over_the_credit_limit\",                                                                  │ │\n",
+       "    │ │       \"name\": \"over-the-credit-limit\",                                                                  │ │\n",
        "    │ │       \"explanation\": \"Over-the-Credit-Limit None\",                                                      │ │\n",
        "    │ │       \"value\": 0.0                                                                                      │ │\n",
        "    │ │     },                                                                                                  │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 8,                                                                                       │ │\n",
-       "    │ │       \"name\": \"return_payment\",                                                                         │ │\n",
+       "    │ │       \"name\": \"return payment\",                                                                         │ │\n",
        "    │ │       \"explanation\": \"Return Payment Up to $40.\",                                                       │ │\n",
        "    │ │       \"value\": 0.0                                                                                      │ │\n",
        "    │ │     },                                                                                                  │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 9,                                                                                       │ │\n",
-       "    │ │       \"name\": \"return_check\",                                                                           │ │\n",
+       "    │ │       \"name\": \"return check\",                                                                           │ │\n",
        "    │ │       \"explanation\": \"Return Check None\",                                                               │ │\n",
        "    │ │       \"value\": 0.0                                                                                      │ │\n",
        "    │ │     }                                                                                                   │ │\n",
@@ -1004,19 +1004,30 @@
        "    │ │     'fees': [                                                                                           │ │\n",
        "    │ │         {                                                                                               │ │\n",
        "    │ │             'index': 1,                                                                                 │ │\n",
-       "    │ │             'name': 'annual_membership',                                                                │ │\n",
+       "    │ │             'name': 'annual membership',                                                                │ │\n",
        "    │ │             'explanation': 'Annual Membership Fee',                                                     │ │\n",
        "    │ │             'value': 0.0                                                                                │ │\n",
        "    │ │         },                                                                                              │ │\n",
        "    │ │         {                                                                                               │ │\n",
        "    │ │             'index': 2,                                                                                 │ │\n",
-       "    │ │             'name': 'my_chase_plan',                                                                    │ │\n",
+       "    │ │             'name': FieldReAsk(                                                                         │ │\n",
+       "    │ │                 incorrect_value='my chase plan',                                                        │ │\n",
+       "    │ │                 fail_results=[                                                                          │ │\n",
+       "    │ │                     FailResult(                                                                         │ │\n",
+       "    │ │                         outcome='fail',                                                                 │ │\n",
+       "    │ │                         metadata=None,                                                                  │ │\n",
+       "    │ │                         error_message='must be exactly two words',                                      │ │\n",
+       "    │ │                         fix_value='my chase'                                                            │ │\n",
+       "    │ │                     )                                                                                   │ │\n",
+       "    │ │                 ],                                                                                      │ │\n",
+       "    │ │                 path=['fees', 1, 'name']                                                                │ │\n",
+       "    │ │             ),                                                                                          │ │\n",
        "    │ │             'explanation': 'My Chase Plan Fee (fixed finance charge)',                                  │ │\n",
        "    │ │             'value': 1.72                                                                               │ │\n",
        "    │ │         },                                                                                              │ │\n",
        "    │ │         {                                                                                               │ │\n",
        "    │ │             'index': 3,                                                                                 │ │\n",
-       "    │ │             'name': 'balance_transfers',                                                                │ │\n",
+       "    │ │             'name': 'balance transfers',                                                                │ │\n",
        "    │ │             'explanation': 'Balance Transfers Intro fee of either $5 or 3% of the amount of each        │ │\n",
        "    │ │ transfer, whichever is greater, on transfers made within 60 days of account opening. After that: Either │ │\n",
        "    │ │ $5 or 5% of the amount of each transfer, whichever is greater.',                                        │ │\n",
@@ -1024,38 +1035,49 @@
        "    │ │         },                                                                                              │ │\n",
        "    │ │         {                                                                                               │ │\n",
        "    │ │             'index': 4,                                                                                 │ │\n",
-       "    │ │             'name': 'cash_advances',                                                                    │ │\n",
+       "    │ │             'name': 'cash advances',                                                                    │ │\n",
        "    │ │             'explanation': 'Either $10 or 5% of the amount of each transaction, whichever is greater.', │ │\n",
        "    │ │             'value': 5.0                                                                                │ │\n",
        "    │ │         },                                                                                              │ │\n",
        "    │ │         {                                                                                               │ │\n",
        "    │ │             'index': 5,                                                                                 │ │\n",
-       "    │ │             'name': 'foreign_transactions',                                                             │ │\n",
+       "    │ │             'name': 'foreign transactions',                                                             │ │\n",
        "    │ │             'explanation': 'Foreign Transactions 3% of the amount of each transaction in U.S.           │ │\n",
        "    │ │ dollars.',                                                                                              │ │\n",
        "    │ │             'value': 3.0                                                                                │ │\n",
        "    │ │         },                                                                                              │ │\n",
        "    │ │         {                                                                                               │ │\n",
        "    │ │             'index': 6,                                                                                 │ │\n",
-       "    │ │             'name': 'late_payment',                                                                     │ │\n",
+       "    │ │             'name': 'late payment',                                                                     │ │\n",
        "    │ │             'explanation': 'Late Payment Up to $40.',                                                   │ │\n",
        "    │ │             'value': 0.0                                                                                │ │\n",
        "    │ │         },                                                                                              │ │\n",
        "    │ │         {                                                                                               │ │\n",
        "    │ │             'index': 7,                                                                                 │ │\n",
-       "    │ │             'name': 'over_the_credit_limit',                                                            │ │\n",
+       "    │ │             'name': FieldReAsk(                                                                         │ │\n",
+       "    │ │                 incorrect_value='over-the-credit-limit',                                                │ │\n",
+       "    │ │                 fail_results=[                                                                          │ │\n",
+       "    │ │                     FailResult(                                                                         │ │\n",
+       "    │ │                         outcome='fail',                                                                 │ │\n",
+       "    │ │                         metadata=None,                                                                  │ │\n",
+       "    │ │                         error_message='must be exactly two words',                                      │ │\n",
+       "    │ │                         fix_value='over the'                                                            │ │\n",
+       "    │ │                     )                                                                                   │ │\n",
+       "    │ │                 ],                                                                                      │ │\n",
+       "    │ │                 path=['fees', 6, 'name']                                                                │ │\n",
+       "    │ │             ),                                                                                          │ │\n",
        "    │ │             'explanation': 'Over-the-Credit-Limit None',                                                │ │\n",
        "    │ │             'value': 0.0                                                                                │ │\n",
        "    │ │         },                                                                                              │ │\n",
        "    │ │         {                                                                                               │ │\n",
        "    │ │             'index': 8,                                                                                 │ │\n",
-       "    │ │             'name': 'return_payment',                                                                   │ │\n",
+       "    │ │             'name': 'return payment',                                                                   │ │\n",
        "    │ │             'explanation': 'Return Payment Up to $40.',                                                 │ │\n",
        "    │ │             'value': 0.0                                                                                │ │\n",
        "    │ │         },                                                                                              │ │\n",
        "    │ │         {                                                                                               │ │\n",
        "    │ │             'index': 9,                                                                                 │ │\n",
-       "    │ │             'name': 'return_check',                                                                     │ │\n",
+       "    │ │             'name': 'return check',                                                                     │ │\n",
        "    │ │             'explanation': 'Return Check None',                                                         │ │\n",
        "    │ │             'value': 0.0                                                                                │ │\n",
        "    │ │         }                                                                                               │ │\n",
@@ -1380,7 +1402,7 @@
        "│   │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        outcome='fail',\u001b[0m\u001b[48;2;240;255;240m                                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "│   │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        metadata=None,\u001b[0m\u001b[48;2;240;255;240m                                                                 \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "│   │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        error_message='must be exactly two words',\u001b[0m\u001b[48;2;240;255;240m                                     \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "│   │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        fix_value='over-the-credit-limit'\u001b[0m\u001b[48;2;240;255;240m                                              \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "│   │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        fix_value='over the'\u001b[0m\u001b[48;2;240;255;240m                                                           \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "│   │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                    )\u001b[0m\u001b[48;2;240;255;240m                                                                                  \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "│   │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                ],\u001b[0m\u001b[48;2;240;255;240m                                                                                     \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "│   │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                path=['fees', 6, 'name']\u001b[0m\u001b[48;2;240;255;240m                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
@@ -1545,19 +1567,19 @@
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m  \"fees\": [\u001b[0m\u001b[48;2;245;245;220m                                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 1,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"annual_membership\",\u001b[0m\u001b[48;2;245;245;220m                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"annual membership\",\u001b[0m\u001b[48;2;245;245;220m                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"Annual Membership Fee\",\u001b[0m\u001b[48;2;245;245;220m                                                          \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"value\": 0.0\u001b[0m\u001b[48;2;245;245;220m                                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    },\u001b[0m\u001b[48;2;245;245;220m                                                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 2,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"my_chase_plan\",\u001b[0m\u001b[48;2;245;245;220m                                                                         \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"my chase plan\",\u001b[0m\u001b[48;2;245;245;220m                                                                         \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"My Chase Plan Fee (fixed finance charge)\",\u001b[0m\u001b[48;2;245;245;220m                                       \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"value\": 1.72\u001b[0m\u001b[48;2;245;245;220m                                                                                    \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    },\u001b[0m\u001b[48;2;245;245;220m                                                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 3,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"balance_transfers\",\u001b[0m\u001b[48;2;245;245;220m                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"balance transfers\",\u001b[0m\u001b[48;2;245;245;220m                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"Balance Transfers Intro fee of either $5 or 3% of the amount of each transfer, \u001b[0m\u001b[48;2;245;245;220m  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220mwhichever is greater, on transfers made within 60 days of account opening. After that: Either $5 or 5% \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220mof the amount of each transfer, whichever is greater.\",\u001b[0m\u001b[48;2;245;245;220m                                                \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
@@ -1565,37 +1587,37 @@
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    },\u001b[0m\u001b[48;2;245;245;220m                                                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 4,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"cash_advances\",\u001b[0m\u001b[48;2;245;245;220m                                                                         \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"cash advances\",\u001b[0m\u001b[48;2;245;245;220m                                                                         \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"Either $10 or 5% of the amount of each transaction, whichever is greater.\",\u001b[0m\u001b[48;2;245;245;220m      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"value\": 5.0\u001b[0m\u001b[48;2;245;245;220m                                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    },\u001b[0m\u001b[48;2;245;245;220m                                                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 5,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"foreign_transactions\",\u001b[0m\u001b[48;2;245;245;220m                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"foreign transactions\",\u001b[0m\u001b[48;2;245;245;220m                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"Foreign Transactions 3% of the amount of each transaction in U.S. dollars.\",\u001b[0m\u001b[48;2;245;245;220m     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"value\": 3.0\u001b[0m\u001b[48;2;245;245;220m                                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    },\u001b[0m\u001b[48;2;245;245;220m                                                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 6,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"late_payment\",\u001b[0m\u001b[48;2;245;245;220m                                                                          \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"late payment\",\u001b[0m\u001b[48;2;245;245;220m                                                                          \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"Late Payment Up to $40.\",\u001b[0m\u001b[48;2;245;245;220m                                                        \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"value\": 0.0\u001b[0m\u001b[48;2;245;245;220m                                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    },\u001b[0m\u001b[48;2;245;245;220m                                                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 7,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"over_the_credit_limit\",\u001b[0m\u001b[48;2;245;245;220m                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"over-the-credit-limit\",\u001b[0m\u001b[48;2;245;245;220m                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"Over-the-Credit-Limit None\",\u001b[0m\u001b[48;2;245;245;220m                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"value\": 0.0\u001b[0m\u001b[48;2;245;245;220m                                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    },\u001b[0m\u001b[48;2;245;245;220m                                                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 8,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"return_payment\",\u001b[0m\u001b[48;2;245;245;220m                                                                        \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"return payment\",\u001b[0m\u001b[48;2;245;245;220m                                                                        \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"Return Payment Up to $40.\",\u001b[0m\u001b[48;2;245;245;220m                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"value\": 0.0\u001b[0m\u001b[48;2;245;245;220m                                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    },\u001b[0m\u001b[48;2;245;245;220m                                                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 9,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"return_check\",\u001b[0m\u001b[48;2;245;245;220m                                                                          \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"return check\",\u001b[0m\u001b[48;2;245;245;220m                                                                          \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"Return Check None\",\u001b[0m\u001b[48;2;245;245;220m                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"value\": 0.0\u001b[0m\u001b[48;2;245;245;220m                                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    }\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
@@ -1627,19 +1649,30 @@
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m    'fees': [\u001b[0m\u001b[48;2;240;255;240m                                                                                          \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {\u001b[0m\u001b[48;2;240;255;240m                                                                                              \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'index': 1,\u001b[0m\u001b[48;2;240;255;240m                                                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'annual_membership',\u001b[0m\u001b[48;2;240;255;240m                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'annual membership',\u001b[0m\u001b[48;2;240;255;240m                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'explanation': 'Annual Membership Fee',\u001b[0m\u001b[48;2;240;255;240m                                                    \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'value': 0.0\u001b[0m\u001b[48;2;240;255;240m                                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        },\u001b[0m\u001b[48;2;240;255;240m                                                                                             \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {\u001b[0m\u001b[48;2;240;255;240m                                                                                              \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'index': 2,\u001b[0m\u001b[48;2;240;255;240m                                                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'my_chase_plan',\u001b[0m\u001b[48;2;240;255;240m                                                                   \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': FieldReAsk(\u001b[0m\u001b[48;2;240;255;240m                                                                        \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                incorrect_value='my chase plan',\u001b[0m\u001b[48;2;240;255;240m                                                       \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                fail_results=[\u001b[0m\u001b[48;2;240;255;240m                                                                         \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                    FailResult(\u001b[0m\u001b[48;2;240;255;240m                                                                        \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        outcome='fail',\u001b[0m\u001b[48;2;240;255;240m                                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        metadata=None,\u001b[0m\u001b[48;2;240;255;240m                                                                 \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        error_message='must be exactly two words',\u001b[0m\u001b[48;2;240;255;240m                                     \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        fix_value='my chase'\u001b[0m\u001b[48;2;240;255;240m                                                           \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                    )\u001b[0m\u001b[48;2;240;255;240m                                                                                  \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                ],\u001b[0m\u001b[48;2;240;255;240m                                                                                     \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                path=['fees', 1, 'name']\u001b[0m\u001b[48;2;240;255;240m                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            ),\u001b[0m\u001b[48;2;240;255;240m                                                                                         \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'explanation': 'My Chase Plan Fee (fixed finance charge)',\u001b[0m\u001b[48;2;240;255;240m                                 \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'value': 1.72\u001b[0m\u001b[48;2;240;255;240m                                                                              \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        },\u001b[0m\u001b[48;2;240;255;240m                                                                                             \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {\u001b[0m\u001b[48;2;240;255;240m                                                                                              \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'index': 3,\u001b[0m\u001b[48;2;240;255;240m                                                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'balance_transfers',\u001b[0m\u001b[48;2;240;255;240m                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'balance transfers',\u001b[0m\u001b[48;2;240;255;240m                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'explanation': 'Balance Transfers Intro fee of either $5 or 3% of the amount of each \u001b[0m\u001b[48;2;240;255;240m      \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240mtransfer, whichever is greater, on transfers made within 60 days of account opening. After that: Either\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m$5 or 5% of the amount of each transfer, whichever is greater.',\u001b[0m\u001b[48;2;240;255;240m                                       \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
@@ -1647,38 +1680,49 @@
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        },\u001b[0m\u001b[48;2;240;255;240m                                                                                             \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {\u001b[0m\u001b[48;2;240;255;240m                                                                                              \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'index': 4,\u001b[0m\u001b[48;2;240;255;240m                                                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'cash_advances',\u001b[0m\u001b[48;2;240;255;240m                                                                   \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'cash advances',\u001b[0m\u001b[48;2;240;255;240m                                                                   \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'explanation': 'Either $10 or 5% of the amount of each transaction, whichever is greater.',\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'value': 5.0\u001b[0m\u001b[48;2;240;255;240m                                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        },\u001b[0m\u001b[48;2;240;255;240m                                                                                             \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {\u001b[0m\u001b[48;2;240;255;240m                                                                                              \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'index': 5,\u001b[0m\u001b[48;2;240;255;240m                                                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'foreign_transactions',\u001b[0m\u001b[48;2;240;255;240m                                                            \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'foreign transactions',\u001b[0m\u001b[48;2;240;255;240m                                                            \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'explanation': 'Foreign Transactions 3% of the amount of each transaction in U.S. \u001b[0m\u001b[48;2;240;255;240m         \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240mdollars.',\u001b[0m\u001b[48;2;240;255;240m                                                                                             \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'value': 3.0\u001b[0m\u001b[48;2;240;255;240m                                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        },\u001b[0m\u001b[48;2;240;255;240m                                                                                             \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {\u001b[0m\u001b[48;2;240;255;240m                                                                                              \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'index': 6,\u001b[0m\u001b[48;2;240;255;240m                                                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'late_payment',\u001b[0m\u001b[48;2;240;255;240m                                                                    \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'late payment',\u001b[0m\u001b[48;2;240;255;240m                                                                    \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'explanation': 'Late Payment Up to $40.',\u001b[0m\u001b[48;2;240;255;240m                                                  \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'value': 0.0\u001b[0m\u001b[48;2;240;255;240m                                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        },\u001b[0m\u001b[48;2;240;255;240m                                                                                             \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {\u001b[0m\u001b[48;2;240;255;240m                                                                                              \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'index': 7,\u001b[0m\u001b[48;2;240;255;240m                                                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'over_the_credit_limit',\u001b[0m\u001b[48;2;240;255;240m                                                           \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': FieldReAsk(\u001b[0m\u001b[48;2;240;255;240m                                                                        \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                incorrect_value='over-the-credit-limit',\u001b[0m\u001b[48;2;240;255;240m                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                fail_results=[\u001b[0m\u001b[48;2;240;255;240m                                                                         \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                    FailResult(\u001b[0m\u001b[48;2;240;255;240m                                                                        \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        outcome='fail',\u001b[0m\u001b[48;2;240;255;240m                                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        metadata=None,\u001b[0m\u001b[48;2;240;255;240m                                                                 \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        error_message='must be exactly two words',\u001b[0m\u001b[48;2;240;255;240m                                     \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        fix_value='over the'\u001b[0m\u001b[48;2;240;255;240m                                                           \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                    )\u001b[0m\u001b[48;2;240;255;240m                                                                                  \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                ],\u001b[0m\u001b[48;2;240;255;240m                                                                                     \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                path=['fees', 6, 'name']\u001b[0m\u001b[48;2;240;255;240m                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            ),\u001b[0m\u001b[48;2;240;255;240m                                                                                         \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'explanation': 'Over-the-Credit-Limit None',\u001b[0m\u001b[48;2;240;255;240m                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'value': 0.0\u001b[0m\u001b[48;2;240;255;240m                                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        },\u001b[0m\u001b[48;2;240;255;240m                                                                                             \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {\u001b[0m\u001b[48;2;240;255;240m                                                                                              \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'index': 8,\u001b[0m\u001b[48;2;240;255;240m                                                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'return_payment',\u001b[0m\u001b[48;2;240;255;240m                                                                  \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'return payment',\u001b[0m\u001b[48;2;240;255;240m                                                                  \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'explanation': 'Return Payment Up to $40.',\u001b[0m\u001b[48;2;240;255;240m                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'value': 0.0\u001b[0m\u001b[48;2;240;255;240m                                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        },\u001b[0m\u001b[48;2;240;255;240m                                                                                             \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {\u001b[0m\u001b[48;2;240;255;240m                                                                                              \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'index': 9,\u001b[0m\u001b[48;2;240;255;240m                                                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'return_check',\u001b[0m\u001b[48;2;240;255;240m                                                                    \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'return check',\u001b[0m\u001b[48;2;240;255;240m                                                                    \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'explanation': 'Return Check None',\u001b[0m\u001b[48;2;240;255;240m                                                        \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'value': 0.0\u001b[0m\u001b[48;2;240;255;240m                                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        }\u001b[0m\u001b[48;2;240;255;240m                                                                                              \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
diff --git a/guardrails/utils/reask_utils.py b/guardrails/utils/reask_utils.py
index 566c47234..678a264e0 100644
--- a/guardrails/utils/reask_utils.py
+++ b/guardrails/utils/reask_utils.py
@@ -213,14 +213,15 @@ def sub_reasks_with_fixed_values(value: Any) -> Any:
     Returns:
         The value with ReAsk objects replaced with their fixed values.
     """
-    if isinstance(value, list):
-        for index, item in enumerate(value):
-            value[index] = sub_reasks_with_fixed_values(item)
-    elif isinstance(value, dict):
+    copy = deepcopy(value)
+    if isinstance(copy, list):
+        for index, item in enumerate(copy):
+            copy[index] = sub_reasks_with_fixed_values(item)
+    elif isinstance(copy, dict):
         for dict_key, dict_value in value.items():
-            value[dict_key] = sub_reasks_with_fixed_values(dict_value)
-    elif isinstance(value, FieldReAsk):
+            copy[dict_key] = sub_reasks_with_fixed_values(dict_value)
+    elif isinstance(copy, FieldReAsk):
         # TODO handle multiple fail results
-        value = value.fail_results[0].fix_value
+        copy = copy.fail_results[0].fix_value
 
-    return value
+    return copy
diff --git a/guardrails/validators/two_words.py b/guardrails/validators/two_words.py
index 3ae260da7..f2182167e 100644
--- a/guardrails/validators/two_words.py
+++ b/guardrails/validators/two_words.py
@@ -1,5 +1,7 @@
 from typing import Any, Dict
 
+from pydash.strings import words as _words
+
 from guardrails.logger import logger
 from guardrails.validator_base import (
     FailResult,
@@ -23,13 +25,24 @@ class TwoWords(Validator):
     | Programmatic fix              | Pick the first two words.         |
     """
 
+    def _get_fix_value(self, value: str) -> str:
+        words = value.split()
+        if len(words) == 1:
+            words = _words(value)
+
+        if len(words) == 1:
+            value = f"{value} {value}"
+            words = value.split()
+
+        return " ".join(words[:2])
+
     def validate(self, value: Any, metadata: Dict) -> ValidationResult:
         logger.debug(f"Validating {value} is two words...")
 
         if len(value.split()) != 2:
             return FailResult(
                 error_message="must be exactly two words",
-                fix_value=" ".join(value.split()[:2]),
+                fix_value=self._get_fix_value(str(value)),
             )
 
         return PassResult()
diff --git a/poetry.lock b/poetry.lock
index 979c7f66e..01f35d153 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -3803,6 +3803,23 @@ files = [
 [package.dependencies]
 typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0"
 
+[[package]]
+name = "pydash"
+version = "7.0.6"
+description = "The kitchen sink of Python utility libraries for doing \"stuff\" in a functional way. Based on the Lo-Dash Javascript library."
+optional = false
+python-versions = ">=3.7"
+files = [
+    {file = "pydash-7.0.6-py3-none-any.whl", hash = "sha256:10e506935953fde4b0d6fe21a88e17783cd1479256ae96f285b5f89063b4efd6"},
+    {file = "pydash-7.0.6.tar.gz", hash = "sha256:7d9df7e9f36f2bbb08316b609480e7c6468185473a21bdd8e65dda7915565a26"},
+]
+
+[package.dependencies]
+typing-extensions = ">=3.10,<4.6.0 || >4.6.0"
+
+[package.extras]
+dev = ["Sphinx", "black", "build", "coverage", "docformatter", "flake8", "flake8-black", "flake8-bugbear", "flake8-isort", "furo", "importlib-metadata (<5)", "invoke", "isort", "mypy", "pylint", "pytest", "pytest-cov", "pytest-mypy-testing", "sphinx-autodoc-typehints", "tox", "twine", "wheel"]
+
 [[package]]
 name = "pyflakes"
 version = "2.5.0"
@@ -6640,4 +6657,4 @@ vectordb = ["faiss-cpu", "numpy", "tiktoken"]
 [metadata]
 lock-version = "2.0"
 python-versions = "^3.8"
-content-hash = "6915b83d33c66b3798ca0cd4417cd179fa9ed04844f0656490abfa7ee2b61b51"
+content-hash = "bde59025d3968af217f47e87f91bc76fb34288d781f8450ed8692e1816f8efe4"
diff --git a/pyproject.toml b/pyproject.toml
index b4d934da8..5bb1c109e 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -47,6 +47,7 @@ anthropic = {version = "^0.7.2", optional = true}
 torch = {version = "^2.1.1", optional = true}
 guardrails-ai-unbabel-comet = {version = "^2.2.1", optional = true}
 huggingface_hub = {version = "^0.16.4", optional = true}
+pydash = "^7.0.6"
 
 
 [tool.poetry.extras]
diff --git a/tests/integration_tests/test_assets/entity_extraction/validated_output_fix.py b/tests/integration_tests/test_assets/entity_extraction/validated_output_fix.py
index d28844dba..b3aa08160 100644
--- a/tests/integration_tests/test_assets/entity_extraction/validated_output_fix.py
+++ b/tests/integration_tests/test_assets/entity_extraction/validated_output_fix.py
@@ -39,7 +39,7 @@
         },
         {
             "index": 7,
-            "name": "over-the-credit-limit",
+            "name": "over the",
             "explanation": "Over-the-Credit-Limit None",
             "value": 0,
         },
diff --git a/tests/integration_tests/test_assets/entity_extraction/validated_output_reask_1.py b/tests/integration_tests/test_assets/entity_extraction/validated_output_reask_1.py
index 22b8d08e2..7f66f781b 100644
--- a/tests/integration_tests/test_assets/entity_extraction/validated_output_reask_1.py
+++ b/tests/integration_tests/test_assets/entity_extraction/validated_output_reask_1.py
@@ -56,7 +56,7 @@
                 fail_results=[
                     FailResult(
                         error_message="must be exactly two words",
-                        fix_value="over-the-credit-limit",
+                        fix_value="over the",
                     )
                 ],
                 path=["fees", 6, "name"],
diff --git a/tests/integration_tests/test_assets/entity_extraction/validated_output_reask_2.py b/tests/integration_tests/test_assets/entity_extraction/validated_output_reask_2.py
index 62a28a421..3ef242077 100644
--- a/tests/integration_tests/test_assets/entity_extraction/validated_output_reask_2.py
+++ b/tests/integration_tests/test_assets/entity_extraction/validated_output_reask_2.py
@@ -39,7 +39,7 @@
         },
         {
             "index": 7,
-            "name": "over-the-credit-limit",
+            "name": "over the",
             "explanation": "Over-the-Credit-Limit None",
             "value": 0,
         },
diff --git a/tests/integration_tests/test_assets/entity_extraction/validated_output_skeleton_reask_2.py b/tests/integration_tests/test_assets/entity_extraction/validated_output_skeleton_reask_2.py
index 5d6f94168..ed07b56c9 100644
--- a/tests/integration_tests/test_assets/entity_extraction/validated_output_skeleton_reask_2.py
+++ b/tests/integration_tests/test_assets/entity_extraction/validated_output_skeleton_reask_2.py
@@ -1,15 +1,15 @@
 # flake8: noqa: E501
 VALIDATED_OUTPUT_SKELETON_REASK_2 = {
     "fees": [
-        {"name": "annual_membership_fee", "explanation": "", "value": 0.0},
-        {"name": "my_chase_plan_fee", "explanation": "", "value": 1.72},
-        {"name": "balance_transfers", "explanation": "", "value": 5.0},
-        {"name": "cash_advances", "explanation": "", "value": 5.0},
-        {"name": "foreign_transactions", "explanation": "", "value": 3.0},
-        {"name": "late_payment", "explanation": "", "value": 0.0},
-        {"name": "over-the-credit-limit", "explanation": "", "value": 0.0},
-        {"name": "return_payment", "explanation": "", "value": 0.0},
-        {"name": "return_check", "explanation": "", "value": 0.0},
+        {"name": "annual membership", "explanation": "", "value": 0.0},
+        {"name": "my chase", "explanation": "", "value": 1.72},
+        {"name": "balance transfers", "explanation": "", "value": 5.0},
+        {"name": "cash advances", "explanation": "", "value": 5.0},
+        {"name": "foreign transactions", "explanation": "", "value": 3.0},
+        {"name": "late payment", "explanation": "", "value": 0.0},
+        {"name": "over the", "explanation": "", "value": 0.0},
+        {"name": "return payment", "explanation": "", "value": 0.0},
+        {"name": "return check", "explanation": "", "value": 0.0},
     ],
     "interest_rates": {
         "purchase": {
diff --git a/tests/unit_tests/test_validators.py b/tests/unit_tests/test_validators.py
index 920bfe8f2..abcc126b2 100644
--- a/tests/unit_tests/test_validators.py
+++ b/tests/unit_tests/test_validators.py
@@ -630,7 +630,7 @@ def custom_refrain_on_fail_handler(value: Any, fail_results: List[FailResult]):
                 fail_results=[
                     FailResult(
                         error_message="must be exactly two words",
-                        fix_value="dog",
+                        fix_value="dog dog",
                     )
                 ],
             ),
diff --git a/tests/unit_tests/validators/test_two_words.py b/tests/unit_tests/validators/test_two_words.py
new file mode 100644
index 000000000..f799705c1
--- /dev/null
+++ b/tests/unit_tests/validators/test_two_words.py
@@ -0,0 +1,52 @@
+import pytest
+
+from guardrails.validators import TwoWords, PassResult, FailResult, ValidationResult
+
+
+def test_two_words_happy_path():
+    validator = TwoWords()
+
+    result: PassResult = validator.validate("Hello there", {})
+
+    assert result.outcome == "pass"
+
+@pytest.mark.parametrize(
+    "input, expected_output",
+    [
+        (
+            "Hello there general",
+            "Hello there"
+        ),
+        (
+            "hello-there-general",
+            "hello there"
+        ),
+        (
+            "hello_there_general",
+            "hello there"
+        ),
+        (
+            "helloThereGeneral",
+            "hello There"
+        ),
+        (
+            "HelloThereGeneral",
+            "Hello There"
+        ),
+        (
+            "hello.there.general",
+            "hello there"
+        ),
+        (
+            "hello",
+            "hello hello"
+        )
+    ]
+)
+def test_two_words_failures(input, expected_output):
+    validator = TwoWords()
+
+    result: FailResult = validator.validate(input, {})
+
+    assert result.outcome == "fail"
+    assert result.fix_value == expected_output
\ No newline at end of file

From e4f60229eee9bb938382132391a9de1c7340e36f Mon Sep 17 00:00:00 2001
From: Caleb Courier 
Date: Wed, 13 Dec 2023 14:57:49 -0600
Subject: [PATCH 15/20] update notebook, fix panel display

---
 docs/examples/extracting_entities.ipynb | 88 +++++++------------------
 guardrails/classes/history/call.py      | 12 ++++
 2 files changed, 34 insertions(+), 66 deletions(-)

diff --git a/docs/examples/extracting_entities.ipynb b/docs/examples/extracting_entities.ipynb
index 69366243b..af51b17f1 100644
--- a/docs/examples/extracting_entities.ipynb
+++ b/docs/examples/extracting_entities.ipynb
@@ -922,19 +922,19 @@
        "    │ │   \"fees\": [                                                                                             │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 1,                                                                                       │ │\n",
-       "    │ │       \"name\": \"annual membership\",                                                                      │ │\n",
+       "    │ │       \"name\": \"annual_membership\",                                                                      │ │\n",
        "    │ │       \"explanation\": \"Annual Membership Fee\",                                                           │ │\n",
        "    │ │       \"value\": 0.0                                                                                      │ │\n",
        "    │ │     },                                                                                                  │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 2,                                                                                       │ │\n",
-       "    │ │       \"name\": \"my chase plan\",                                                                          │ │\n",
+       "    │ │       \"name\": \"my_chase_plan\",                                                                          │ │\n",
        "    │ │       \"explanation\": \"My Chase Plan Fee (fixed finance charge)\",                                        │ │\n",
        "    │ │       \"value\": 1.72                                                                                     │ │\n",
        "    │ │     },                                                                                                  │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 3,                                                                                       │ │\n",
-       "    │ │       \"name\": \"balance transfers\",                                                                      │ │\n",
+       "    │ │       \"name\": \"balance_transfers\",                                                                      │ │\n",
        "    │ │       \"explanation\": \"Balance Transfers Intro fee of either $5 or 3% of the amount of each transfer,    │ │\n",
        "    │ │ whichever is greater, on transfers made within 60 days of account opening. After that: Either $5 or 5%  │ │\n",
        "    │ │ of the amount of each transfer, whichever is greater.\",                                                 │ │\n",
@@ -942,37 +942,37 @@
        "    │ │     },                                                                                                  │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 4,                                                                                       │ │\n",
-       "    │ │       \"name\": \"cash advances\",                                                                          │ │\n",
+       "    │ │       \"name\": \"cash_advances\",                                                                          │ │\n",
        "    │ │       \"explanation\": \"Either $10 or 5% of the amount of each transaction, whichever is greater.\",       │ │\n",
        "    │ │       \"value\": 5.0                                                                                      │ │\n",
        "    │ │     },                                                                                                  │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 5,                                                                                       │ │\n",
-       "    │ │       \"name\": \"foreign transactions\",                                                                   │ │\n",
+       "    │ │       \"name\": \"foreign_transactions\",                                                                   │ │\n",
        "    │ │       \"explanation\": \"Foreign Transactions 3% of the amount of each transaction in U.S. dollars.\",      │ │\n",
        "    │ │       \"value\": 3.0                                                                                      │ │\n",
        "    │ │     },                                                                                                  │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 6,                                                                                       │ │\n",
-       "    │ │       \"name\": \"late payment\",                                                                           │ │\n",
+       "    │ │       \"name\": \"late_payment\",                                                                           │ │\n",
        "    │ │       \"explanation\": \"Late Payment Up to $40.\",                                                         │ │\n",
        "    │ │       \"value\": 0.0                                                                                      │ │\n",
        "    │ │     },                                                                                                  │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 7,                                                                                       │ │\n",
-       "    │ │       \"name\": \"over-the-credit-limit\",                                                                  │ │\n",
+       "    │ │       \"name\": \"over_the_credit_limit\",                                                                  │ │\n",
        "    │ │       \"explanation\": \"Over-the-Credit-Limit None\",                                                      │ │\n",
        "    │ │       \"value\": 0.0                                                                                      │ │\n",
        "    │ │     },                                                                                                  │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 8,                                                                                       │ │\n",
-       "    │ │       \"name\": \"return payment\",                                                                         │ │\n",
+       "    │ │       \"name\": \"return_payment\",                                                                         │ │\n",
        "    │ │       \"explanation\": \"Return Payment Up to $40.\",                                                       │ │\n",
        "    │ │       \"value\": 0.0                                                                                      │ │\n",
        "    │ │     },                                                                                                  │ │\n",
        "    │ │     {                                                                                                   │ │\n",
        "    │ │       \"index\": 9,                                                                                       │ │\n",
-       "    │ │       \"name\": \"return check\",                                                                           │ │\n",
+       "    │ │       \"name\": \"return_check\",                                                                           │ │\n",
        "    │ │       \"explanation\": \"Return Check None\",                                                               │ │\n",
        "    │ │       \"value\": 0.0                                                                                      │ │\n",
        "    │ │     }                                                                                                   │ │\n",
@@ -1010,18 +1010,7 @@
        "    │ │         },                                                                                              │ │\n",
        "    │ │         {                                                                                               │ │\n",
        "    │ │             'index': 2,                                                                                 │ │\n",
-       "    │ │             'name': FieldReAsk(                                                                         │ │\n",
-       "    │ │                 incorrect_value='my chase plan',                                                        │ │\n",
-       "    │ │                 fail_results=[                                                                          │ │\n",
-       "    │ │                     FailResult(                                                                         │ │\n",
-       "    │ │                         outcome='fail',                                                                 │ │\n",
-       "    │ │                         metadata=None,                                                                  │ │\n",
-       "    │ │                         error_message='must be exactly two words',                                      │ │\n",
-       "    │ │                         fix_value='my chase'                                                            │ │\n",
-       "    │ │                     )                                                                                   │ │\n",
-       "    │ │                 ],                                                                                      │ │\n",
-       "    │ │                 path=['fees', 1, 'name']                                                                │ │\n",
-       "    │ │             ),                                                                                          │ │\n",
+       "    │ │             'name': 'my chase',                                                                         │ │\n",
        "    │ │             'explanation': 'My Chase Plan Fee (fixed finance charge)',                                  │ │\n",
        "    │ │             'value': 1.72                                                                               │ │\n",
        "    │ │         },                                                                                              │ │\n",
@@ -1054,18 +1043,7 @@
        "    │ │         },                                                                                              │ │\n",
        "    │ │         {                                                                                               │ │\n",
        "    │ │             'index': 7,                                                                                 │ │\n",
-       "    │ │             'name': FieldReAsk(                                                                         │ │\n",
-       "    │ │                 incorrect_value='over-the-credit-limit',                                                │ │\n",
-       "    │ │                 fail_results=[                                                                          │ │\n",
-       "    │ │                     FailResult(                                                                         │ │\n",
-       "    │ │                         outcome='fail',                                                                 │ │\n",
-       "    │ │                         metadata=None,                                                                  │ │\n",
-       "    │ │                         error_message='must be exactly two words',                                      │ │\n",
-       "    │ │                         fix_value='over the'                                                            │ │\n",
-       "    │ │                     )                                                                                   │ │\n",
-       "    │ │                 ],                                                                                      │ │\n",
-       "    │ │                 path=['fees', 6, 'name']                                                                │ │\n",
-       "    │ │             ),                                                                                          │ │\n",
+       "    │ │             'name': 'over the',                                                                         │ │\n",
        "    │ │             'explanation': 'Over-the-Credit-Limit None',                                                │ │\n",
        "    │ │             'value': 0.0                                                                                │ │\n",
        "    │ │         },                                                                                              │ │\n",
@@ -1567,19 +1545,19 @@
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m  \"fees\": [\u001b[0m\u001b[48;2;245;245;220m                                                                                            \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 1,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"annual membership\",\u001b[0m\u001b[48;2;245;245;220m                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"annual_membership\",\u001b[0m\u001b[48;2;245;245;220m                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"Annual Membership Fee\",\u001b[0m\u001b[48;2;245;245;220m                                                          \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"value\": 0.0\u001b[0m\u001b[48;2;245;245;220m                                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    },\u001b[0m\u001b[48;2;245;245;220m                                                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 2,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"my chase plan\",\u001b[0m\u001b[48;2;245;245;220m                                                                         \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"my_chase_plan\",\u001b[0m\u001b[48;2;245;245;220m                                                                         \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"My Chase Plan Fee (fixed finance charge)\",\u001b[0m\u001b[48;2;245;245;220m                                       \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"value\": 1.72\u001b[0m\u001b[48;2;245;245;220m                                                                                    \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    },\u001b[0m\u001b[48;2;245;245;220m                                                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 3,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"balance transfers\",\u001b[0m\u001b[48;2;245;245;220m                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"balance_transfers\",\u001b[0m\u001b[48;2;245;245;220m                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"Balance Transfers Intro fee of either $5 or 3% of the amount of each transfer, \u001b[0m\u001b[48;2;245;245;220m  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220mwhichever is greater, on transfers made within 60 days of account opening. After that: Either $5 or 5% \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220mof the amount of each transfer, whichever is greater.\",\u001b[0m\u001b[48;2;245;245;220m                                                \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
@@ -1587,37 +1565,37 @@
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    },\u001b[0m\u001b[48;2;245;245;220m                                                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 4,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"cash advances\",\u001b[0m\u001b[48;2;245;245;220m                                                                         \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"cash_advances\",\u001b[0m\u001b[48;2;245;245;220m                                                                         \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"Either $10 or 5% of the amount of each transaction, whichever is greater.\",\u001b[0m\u001b[48;2;245;245;220m      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"value\": 5.0\u001b[0m\u001b[48;2;245;245;220m                                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    },\u001b[0m\u001b[48;2;245;245;220m                                                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 5,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"foreign transactions\",\u001b[0m\u001b[48;2;245;245;220m                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"foreign_transactions\",\u001b[0m\u001b[48;2;245;245;220m                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"Foreign Transactions 3% of the amount of each transaction in U.S. dollars.\",\u001b[0m\u001b[48;2;245;245;220m     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"value\": 3.0\u001b[0m\u001b[48;2;245;245;220m                                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    },\u001b[0m\u001b[48;2;245;245;220m                                                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 6,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"late payment\",\u001b[0m\u001b[48;2;245;245;220m                                                                          \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"late_payment\",\u001b[0m\u001b[48;2;245;245;220m                                                                          \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"Late Payment Up to $40.\",\u001b[0m\u001b[48;2;245;245;220m                                                        \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"value\": 0.0\u001b[0m\u001b[48;2;245;245;220m                                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    },\u001b[0m\u001b[48;2;245;245;220m                                                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 7,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"over-the-credit-limit\",\u001b[0m\u001b[48;2;245;245;220m                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"over_the_credit_limit\",\u001b[0m\u001b[48;2;245;245;220m                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"Over-the-Credit-Limit None\",\u001b[0m\u001b[48;2;245;245;220m                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"value\": 0.0\u001b[0m\u001b[48;2;245;245;220m                                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    },\u001b[0m\u001b[48;2;245;245;220m                                                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 8,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"return payment\",\u001b[0m\u001b[48;2;245;245;220m                                                                        \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"return_payment\",\u001b[0m\u001b[48;2;245;245;220m                                                                        \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"Return Payment Up to $40.\",\u001b[0m\u001b[48;2;245;245;220m                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"value\": 0.0\u001b[0m\u001b[48;2;245;245;220m                                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    },\u001b[0m\u001b[48;2;245;245;220m                                                                                                 \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    {\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"index\": 9,\u001b[0m\u001b[48;2;245;245;220m                                                                                      \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
-       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"return check\",\u001b[0m\u001b[48;2;245;245;220m                                                                          \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
+       "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"name\": \"return_check\",\u001b[0m\u001b[48;2;245;245;220m                                                                          \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"explanation\": \"Return Check None\",\u001b[0m\u001b[48;2;245;245;220m                                                              \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m      \"value\": 0.0\u001b[0m\u001b[48;2;245;245;220m                                                                                     \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
        "    │ \u001b[48;2;245;245;220m│\u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m    }\u001b[0m\u001b[48;2;245;245;220m                                                                                                  \u001b[0m\u001b[48;2;245;245;220m \u001b[0m\u001b[48;2;245;245;220m│\u001b[0m │\n",
@@ -1655,18 +1633,7 @@
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        },\u001b[0m\u001b[48;2;240;255;240m                                                                                             \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {\u001b[0m\u001b[48;2;240;255;240m                                                                                              \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'index': 2,\u001b[0m\u001b[48;2;240;255;240m                                                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': FieldReAsk(\u001b[0m\u001b[48;2;240;255;240m                                                                        \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                incorrect_value='my chase plan',\u001b[0m\u001b[48;2;240;255;240m                                                       \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                fail_results=[\u001b[0m\u001b[48;2;240;255;240m                                                                         \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                    FailResult(\u001b[0m\u001b[48;2;240;255;240m                                                                        \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        outcome='fail',\u001b[0m\u001b[48;2;240;255;240m                                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        metadata=None,\u001b[0m\u001b[48;2;240;255;240m                                                                 \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        error_message='must be exactly two words',\u001b[0m\u001b[48;2;240;255;240m                                     \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        fix_value='my chase'\u001b[0m\u001b[48;2;240;255;240m                                                           \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                    )\u001b[0m\u001b[48;2;240;255;240m                                                                                  \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                ],\u001b[0m\u001b[48;2;240;255;240m                                                                                     \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                path=['fees', 1, 'name']\u001b[0m\u001b[48;2;240;255;240m                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            ),\u001b[0m\u001b[48;2;240;255;240m                                                                                         \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'my chase',\u001b[0m\u001b[48;2;240;255;240m                                                                        \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'explanation': 'My Chase Plan Fee (fixed finance charge)',\u001b[0m\u001b[48;2;240;255;240m                                 \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'value': 1.72\u001b[0m\u001b[48;2;240;255;240m                                                                              \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        },\u001b[0m\u001b[48;2;240;255;240m                                                                                             \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
@@ -1699,18 +1666,7 @@
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        },\u001b[0m\u001b[48;2;240;255;240m                                                                                             \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        {\u001b[0m\u001b[48;2;240;255;240m                                                                                              \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'index': 7,\u001b[0m\u001b[48;2;240;255;240m                                                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': FieldReAsk(\u001b[0m\u001b[48;2;240;255;240m                                                                        \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                incorrect_value='over-the-credit-limit',\u001b[0m\u001b[48;2;240;255;240m                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                fail_results=[\u001b[0m\u001b[48;2;240;255;240m                                                                         \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                    FailResult(\u001b[0m\u001b[48;2;240;255;240m                                                                        \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        outcome='fail',\u001b[0m\u001b[48;2;240;255;240m                                                                \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        metadata=None,\u001b[0m\u001b[48;2;240;255;240m                                                                 \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        error_message='must be exactly two words',\u001b[0m\u001b[48;2;240;255;240m                                     \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                        fix_value='over the'\u001b[0m\u001b[48;2;240;255;240m                                                           \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                    )\u001b[0m\u001b[48;2;240;255;240m                                                                                  \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                ],\u001b[0m\u001b[48;2;240;255;240m                                                                                     \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m                path=['fees', 6, 'name']\u001b[0m\u001b[48;2;240;255;240m                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
-       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            ),\u001b[0m\u001b[48;2;240;255;240m                                                                                         \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
+       "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'name': 'over the',\u001b[0m\u001b[48;2;240;255;240m                                                                        \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'explanation': 'Over-the-Credit-Limit None',\u001b[0m\u001b[48;2;240;255;240m                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m            'value': 0.0\u001b[0m\u001b[48;2;240;255;240m                                                                               \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
        "    │ \u001b[48;2;240;255;240m│\u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m        },\u001b[0m\u001b[48;2;240;255;240m                                                                                             \u001b[0m\u001b[48;2;240;255;240m \u001b[0m\u001b[48;2;240;255;240m│\u001b[0m │\n",
diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py
index a9fdf12bf..96da45165 100644
--- a/guardrails/classes/history/call.py
+++ b/guardrails/classes/history/call.py
@@ -2,6 +2,7 @@
 
 from pydantic import Field, PrivateAttr
 from rich.panel import Panel
+from rich.pretty import pretty_repr
 from rich.tree import Tree
 
 from guardrails.classes.generic.stack import Stack
@@ -330,4 +331,15 @@ def tree(self) -> Tree:
         tree = Tree("Logs")
         for i, iteration in enumerate(self.iterations):
             tree.add(Panel(iteration.rich_group, title=f"Step {i}"))
+        
+        # Replace the last Validated Output panel if we applied fixes
+        if self.failed_validations.length > 0 and self.status == pass_status:
+            previous_panels = tree.children[-1].label.renderable._renderables[:-1]
+            validated_outcome_panel = Panel(
+                pretty_repr(self.validated_output),
+                title="Validated Output",
+                style="on #F0FFF0",
+            )
+            tree.children[-1].label.renderable._renderables = previous_panels + (validated_outcome_panel,)
+        
         return tree

From 44697c36ced57d5ed5cade61df43f853bd2224bb Mon Sep 17 00:00:00 2001
From: Caleb Courier 
Date: Wed, 13 Dec 2023 15:00:42 -0600
Subject: [PATCH 16/20] lint

---
 guardrails/classes/history/call.py            |  8 ++--
 tests/unit_tests/validators/test_two_words.py | 42 +++++--------------
 2 files changed, 16 insertions(+), 34 deletions(-)

diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py
index 96da45165..d2804b586 100644
--- a/guardrails/classes/history/call.py
+++ b/guardrails/classes/history/call.py
@@ -331,7 +331,7 @@ def tree(self) -> Tree:
         tree = Tree("Logs")
         for i, iteration in enumerate(self.iterations):
             tree.add(Panel(iteration.rich_group, title=f"Step {i}"))
-        
+
         # Replace the last Validated Output panel if we applied fixes
         if self.failed_validations.length > 0 and self.status == pass_status:
             previous_panels = tree.children[-1].label.renderable._renderables[:-1]
@@ -340,6 +340,8 @@ def tree(self) -> Tree:
                 title="Validated Output",
                 style="on #F0FFF0",
             )
-            tree.children[-1].label.renderable._renderables = previous_panels + (validated_outcome_panel,)
-        
+            tree.children[-1].label.renderable._renderables = previous_panels + (
+                validated_outcome_panel,
+            )
+
         return tree
diff --git a/tests/unit_tests/validators/test_two_words.py b/tests/unit_tests/validators/test_two_words.py
index f799705c1..ab327067b 100644
--- a/tests/unit_tests/validators/test_two_words.py
+++ b/tests/unit_tests/validators/test_two_words.py
@@ -1,6 +1,6 @@
 import pytest
 
-from guardrails.validators import TwoWords, PassResult, FailResult, ValidationResult
+from guardrails.validators import FailResult, PassResult, TwoWords, ValidationResult
 
 
 def test_two_words_happy_path():
@@ -10,38 +10,18 @@ def test_two_words_happy_path():
 
     assert result.outcome == "pass"
 
+
 @pytest.mark.parametrize(
     "input, expected_output",
     [
-        (
-            "Hello there general",
-            "Hello there"
-        ),
-        (
-            "hello-there-general",
-            "hello there"
-        ),
-        (
-            "hello_there_general",
-            "hello there"
-        ),
-        (
-            "helloThereGeneral",
-            "hello There"
-        ),
-        (
-            "HelloThereGeneral",
-            "Hello There"
-        ),
-        (
-            "hello.there.general",
-            "hello there"
-        ),
-        (
-            "hello",
-            "hello hello"
-        )
-    ]
+        ("Hello there general", "Hello there"),
+        ("hello-there-general", "hello there"),
+        ("hello_there_general", "hello there"),
+        ("helloThereGeneral", "hello There"),
+        ("HelloThereGeneral", "Hello There"),
+        ("hello.there.general", "hello there"),
+        ("hello", "hello hello"),
+    ],
 )
 def test_two_words_failures(input, expected_output):
     validator = TwoWords()
@@ -49,4 +29,4 @@ def test_two_words_failures(input, expected_output):
     result: FailResult = validator.validate(input, {})
 
     assert result.outcome == "fail"
-    assert result.fix_value == expected_output
\ No newline at end of file
+    assert result.fix_value == expected_output

From 10cd685ace031b0bf66577c900d0135803ceb7ae Mon Sep 17 00:00:00 2001
From: Caleb Courier 
Date: Wed, 13 Dec 2023 15:03:00 -0600
Subject: [PATCH 17/20] type ignores

---
 guardrails/classes/history/call.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py
index d2804b586..e496356cd 100644
--- a/guardrails/classes/history/call.py
+++ b/guardrails/classes/history/call.py
@@ -334,13 +334,13 @@ def tree(self) -> Tree:
 
         # Replace the last Validated Output panel if we applied fixes
         if self.failed_validations.length > 0 and self.status == pass_status:
-            previous_panels = tree.children[-1].label.renderable._renderables[:-1]
+            previous_panels = tree.children[-1].label.renderable._renderables[:-1]  # type: ignore
             validated_outcome_panel = Panel(
                 pretty_repr(self.validated_output),
                 title="Validated Output",
                 style="on #F0FFF0",
             )
-            tree.children[-1].label.renderable._renderables = previous_panels + (
+            tree.children[-1].label.renderable._renderables = previous_panels + ( # type: ignore
                 validated_outcome_panel,
             )
 

From 6b32d20e7781c009533b1b37d9338d2f4fe22ea8 Mon Sep 17 00:00:00 2001
From: Caleb Courier 
Date: Wed, 13 Dec 2023 15:09:57 -0600
Subject: [PATCH 18/20] make lint and type work together

---
 guardrails/classes/history/call.py            | 10 ++++++++--
 tests/unit_tests/validators/test_two_words.py |  2 +-
 2 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py
index e496356cd..90b01e58b 100644
--- a/guardrails/classes/history/call.py
+++ b/guardrails/classes/history/call.py
@@ -334,13 +334,19 @@ def tree(self) -> Tree:
 
         # Replace the last Validated Output panel if we applied fixes
         if self.failed_validations.length > 0 and self.status == pass_status:
-            previous_panels = tree.children[-1].label.renderable._renderables[:-1]  # type: ignore
+            previous_panels = tree.children[  # type: ignore
+                -1
+            ].label.renderable._renderables[  # type: ignore
+                :-1
+            ]
             validated_outcome_panel = Panel(
                 pretty_repr(self.validated_output),
                 title="Validated Output",
                 style="on #F0FFF0",
             )
-            tree.children[-1].label.renderable._renderables = previous_panels + ( # type: ignore
+            tree.children[
+                -1
+            ].label.renderable._renderables = previous_panels + (  # type: ignore
                 validated_outcome_panel,
             )
 
diff --git a/tests/unit_tests/validators/test_two_words.py b/tests/unit_tests/validators/test_two_words.py
index ab327067b..2a66f4442 100644
--- a/tests/unit_tests/validators/test_two_words.py
+++ b/tests/unit_tests/validators/test_two_words.py
@@ -1,6 +1,6 @@
 import pytest
 
-from guardrails.validators import FailResult, PassResult, TwoWords, ValidationResult
+from guardrails.validators import FailResult, PassResult, TwoWords
 
 
 def test_two_words_happy_path():

From 97ca58ad9717c5d6541a277cd19ef24a6a1924c1 Mon Sep 17 00:00:00 2001
From: Caleb Courier 
Date: Thu, 14 Dec 2023 11:47:47 -0600
Subject: [PATCH 19/20] expose guardrails.errors, fix notebook

---
 docs/examples/input_validation.ipynb | 53 +++++++++++++++++++---------
 guardrails/errors/__init__.py        |  3 ++
 2 files changed, 40 insertions(+), 16 deletions(-)
 create mode 100644 guardrails/errors/__init__.py

diff --git a/docs/examples/input_validation.ipynb b/docs/examples/input_validation.ipynb
index 527168673..82c68cd7b 100644
--- a/docs/examples/input_validation.ipynb
+++ b/docs/examples/input_validation.ipynb
@@ -18,7 +18,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 4,
    "metadata": {
     "is_executing": true
    },
@@ -47,23 +47,34 @@
    "source": [
     "When `fix` is specified as the on-fail handler, the prompt will automatically be amended before calling the LLM.\n",
     "\n",
-    "In any other case (for example, `exception`), a `ValidationException` will be returned in the outcome."
+    "In any other case (for example, `exception`), a `ValidatorError` will be returned in the outcome."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 5,
    "metadata": {
     "is_executing": true
    },
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Validation failed for field with errors: must be exactly two words\n"
+     ]
+    }
+   ],
    "source": [
     "import openai\n",
+    "from guardrails.errors import ValidatorError\n",
     "\n",
-    "outcome = guard(\n",
-    "    openai.ChatCompletion.create,\n",
-    ")\n",
-    "outcome.error"
+    "try:\n",
+    "    guard(\n",
+    "        openai.ChatCompletion.create,\n",
+    "    )\n",
+    "except ValidatorError as e:\n",
+    "    print(e)"
    ]
   },
   {
@@ -75,9 +86,17 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 6,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Validation failed for field with errors: must be exactly two words\n"
+     ]
+    }
+   ],
    "source": [
     "from guardrails.validators import TwoWords\n",
     "from pydantic import BaseModel\n",
@@ -91,11 +110,13 @@
     "guard = Guard.from_pydantic(Pet)\n",
     "guard.with_prompt_validation([TwoWords(on_fail=\"exception\")])\n",
     "\n",
-    "outcome = guard(\n",
-    "    openai.ChatCompletion.create,\n",
-    "    prompt=\"This is not two words\",\n",
-    ")\n",
-    "outcome.error"
+    "try:\n",
+    "    guard(\n",
+    "        openai.ChatCompletion.create,\n",
+    "        prompt=\"This is not two words\",\n",
+    "    )\n",
+    "except ValidatorError as e:\n",
+    "    print(e)"
    ]
   }
  ],
@@ -115,7 +136,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.0"
+   "version": "3.11.6"
   }
  },
  "nbformat": 4,
diff --git a/guardrails/errors/__init__.py b/guardrails/errors/__init__.py
new file mode 100644
index 000000000..392ed17fa
--- /dev/null
+++ b/guardrails/errors/__init__.py
@@ -0,0 +1,3 @@
+from guardrails.validator_base import ValidatorError
+
+__all__ = ["ValidatorError"]

From 98fbfcd0207d6b8240fe6733c7c738c88b99541d Mon Sep 17 00:00:00 2001
From: Caleb Courier 
Date: Thu, 14 Dec 2023 11:52:57 -0600
Subject: [PATCH 20/20] fix on topic notebook error handling

---
 docs/examples/response_is_on_topic.ipynb | 46 +++++++++++++-----------
 1 file changed, 26 insertions(+), 20 deletions(-)

diff --git a/docs/examples/response_is_on_topic.ipynb b/docs/examples/response_is_on_topic.ipynb
index 4fc393d1b..a9b8f0444 100644
--- a/docs/examples/response_is_on_topic.ipynb
+++ b/docs/examples/response_is_on_topic.ipynb
@@ -37,7 +37,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 20,
+   "execution_count": 1,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -54,7 +54,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 21,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -76,7 +76,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 22,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -115,7 +115,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 23,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -129,6 +129,7 @@
    "source": [
     "import guardrails as gd\n",
     "from guardrails.validators import OnTopic\n",
+    "from guardrails.errors import ValidatorError\n",
     "\n",
     "# Create the Guard with the OnTopic Validator\n",
     "guard = gd.Guard.from_string(\n",
@@ -146,11 +147,12 @@
     ")\n",
     "\n",
     "# Test with a given text\n",
-    "output = guard.parse(\n",
-    "    llm_output=text,\n",
-    ")\n",
-    "\n",
-    "print(output.error)"
+    "try:\n",
+    "    guard.parse(\n",
+    "        llm_output=text,\n",
+    "    )\n",
+    "except ValidatorError as e:\n",
+    "    print(e)\n"
    ]
   },
   {
@@ -164,7 +166,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 24,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [
     {
@@ -191,9 +193,12 @@
     ")\n",
     "\n",
     "# Test with a given text\n",
-    "output = guard.parse(llm_output=text)\n",
-    "\n",
-    "print(output.error)"
+    "try:\n",
+    "    guard.parse(\n",
+    "        llm_output=text,\n",
+    "    )\n",
+    "except ValidatorError as e:\n",
+    "    print(e)"
    ]
   },
   {
@@ -207,7 +212,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 25,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -234,11 +239,12 @@
     ")\n",
     "\n",
     "# Test with a given text\n",
-    "output = guard.parse(\n",
-    "    llm_output=text\n",
-    ")\n",
-    "\n",
-    "print(output.error)"
+    "try:\n",
+    "    guard.parse(\n",
+    "        llm_output=text,\n",
+    "    )\n",
+    "except ValidatorError as e:\n",
+    "    print(e)"
    ]
   }
  ],
@@ -258,7 +264,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.11.6"
   }
  },
  "nbformat": 4,