From 6844d420618b9a3c8ef5faf7ba5f3ba9de7811be Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Thu, 14 Mar 2024 13:35:42 -0500 Subject: [PATCH 1/6] valdiation_output -> validation_response --- guardrails/classes/history/call.py | 21 ++++++++++++--- guardrails/classes/history/iteration.py | 19 +++++++++++--- guardrails/classes/history/outputs.py | 20 +++++++++++--- guardrails/classes/validation_outcome.py | 2 +- guardrails/run.py | 20 +++++++------- guardrails/utils/deprecation.py | 6 +++++ tests/integration_tests/test_async.py | 8 +++--- tests/integration_tests/test_guard.py | 20 +++++++------- tests/integration_tests/test_multi_reask.py | 6 ++--- tests/integration_tests/test_pydantic.py | 20 +++++++++----- tests/integration_tests/test_python_rail.py | 2 +- tests/unit_tests/classes/history/test_call.py | 6 ++--- .../classes/history/test_iteration.py | 6 ++--- .../classes/history/test_outputs.py | 4 +-- tests/unit_tests/test_validators.py | 26 +++++++++++++------ 15 files changed, 124 insertions(+), 62 deletions(-) create mode 100644 guardrails/utils/deprecation.py diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py index 0623085fd..2bad3e7ec 100644 --- a/guardrails/classes/history/call.py +++ b/guardrails/classes/history/call.py @@ -4,6 +4,7 @@ from rich.panel import Panel from rich.pretty import pretty_repr from rich.tree import Tree +from typing_extensions import deprecated from guardrails.classes.generic.stack import Stack from guardrails.classes.history.call_inputs import CallInputs @@ -198,8 +199,17 @@ def parsed_outputs(self) -> Stack[Union[str, Dict]]: return Stack(*[i.outputs.parsed_output for i in self.iterations]) @property + @deprecated( + """'Call.validation_output' is deprecated and will be removed in \ +versions 0.5.0 and beyond. Use 'validation_response' instead.""" + ) def validation_output(self) -> Optional[Union[str, Dict, ReAsk]]: - """The cumulative validation output across all current iterations. + return self.validation_response + + @property + def validation_response(self) -> Optional[Union[str, Dict, ReAsk]]: + """The aggregated responses from the validation process across all + iterations with the current call. Could contain ReAsks. """ @@ -240,9 +250,12 @@ def validation_output(self) -> Optional[Union[str, Dict, ReAsk]]: @property def fixed_output(self) -> Optional[Union[str, Dict]]: - """The cumulative validation output across all current iterations with - any automatic fixes applied.""" - return sub_reasks_with_fixed_values(self.validation_output) + """The cumulative output from the validation process across all current + iterations with any automatic fixes applied. + + Could still contain ReAsks if a fix was not available. + """ + return sub_reasks_with_fixed_values(self.validation_response) @property def validated_output(self) -> Optional[Union[str, Dict]]: diff --git a/guardrails/classes/history/iteration.py b/guardrails/classes/history/iteration.py index 0fcb82ef3..bc0717eb4 100644 --- a/guardrails/classes/history/iteration.py +++ b/guardrails/classes/history/iteration.py @@ -5,6 +5,7 @@ from rich.panel import Panel from rich.pretty import pretty_repr from rich.table import Table +from typing_extensions import deprecated from guardrails.classes.generic.stack import Stack from guardrails.classes.history.inputs import Inputs @@ -76,12 +77,24 @@ def parsed_output(self) -> Optional[Union[str, Dict]]: return self.outputs.parsed_output @property + def validation_response(self) -> Optional[Union[ReAsk, str, Dict]]: + """The response from the validation process. + + Could be a combination of valid output and ReAsks + """ + return self.outputs.validation_response + + @property + @deprecated( + """'Iteration.validation_output' is deprecated and will be removed in \ +versions 0.5.0 and beyond. Use 'validation_response' instead.""" + ) def validation_output(self) -> Optional[Union[ReAsk, str, Dict]]: """The output from the validation process. Could be a combination of valid output and ReAsks """ - return self.outputs.validation_output + return self.validation_response @property def validated_output(self) -> Optional[Union[str, Dict]]: @@ -167,7 +180,7 @@ def create_msg_history_table( self.raw_output or "", title="Raw LLM Output", style="on #F5F5DC" ), Panel( - pretty_repr(self.validation_output), + pretty_repr(self.validation_response), title="Validated Output", style="on #F0FFF0", ), @@ -184,7 +197,7 @@ def create_msg_history_table( self.raw_output or "", title="Raw LLM Output", style="on #F5F5DC" ), Panel( - pretty_repr(self.validation_output), + pretty_repr(self.validation_response), title="Validated Output", style="on #F0FFF0", ), diff --git a/guardrails/classes/history/outputs.py b/guardrails/classes/history/outputs.py index 8e009f175..e76be4032 100644 --- a/guardrails/classes/history/outputs.py +++ b/guardrails/classes/history/outputs.py @@ -1,6 +1,7 @@ from typing import Dict, List, Optional, Sequence, Union from pydantic import Field +from typing_extensions import deprecated from guardrails.constants import error_status, fail_status, not_run_status, pass_status from guardrails.utils.llm_response import LLMResponse @@ -22,9 +23,12 @@ class Outputs(ArbitraryModel): "as it was passed into validation.", default=None, ) - validation_output: Optional[Union[str, ReAsk, Dict]] = Field( - description="The output from the validation process.", default=None + validation_response: Optional[Union[str, ReAsk, Dict]] = Field( + description="The response from the validation process.", default=None ) + # validation_output: Optional[Union[str, ReAsk, Dict]] = Field( + # description="The output from the validation process.", default=None + # ) validated_output: Optional[Union[str, Dict]] = Field( description="The valid output after validation." "Could be only a partial structure if field level reasks occur." @@ -54,7 +58,7 @@ def _all_empty(self) -> bool: return ( self.llm_response_info is None and self.parsed_output is None - and self.validation_output is None + and self.validation_response is None and self.validated_output is None and len(self.reasks) == 0 and len(self.validator_logs) == 0 @@ -94,7 +98,15 @@ def status(self) -> str: elif not all_reasks_have_fixes: return fail_status elif self.validated_output is None and isinstance( - self.validation_output, ReAsk + self.validation_response, ReAsk ): return fail_status return pass_status + + @property + @deprecated( + """'Outputs.validation_output' is deprecated and will be removed in \ +versions 0.5.0 and beyond. Use 'validation_response' instead.""" + ) + def validation_output(self) -> Optional[Union[str, ReAsk, Dict]]: + return self.validation_response diff --git a/guardrails/classes/validation_outcome.py b/guardrails/classes/validation_outcome.py index 4f9df16ae..fae18fe90 100644 --- a/guardrails/classes/validation_outcome.py +++ b/guardrails/classes/validation_outcome.py @@ -51,7 +51,7 @@ class ValidationOutcome(ArbitraryModel, Generic[OT]): def from_guard_history(cls, call: Call): """Create a ValidationOutcome from a history Call object.""" last_iteration = call.iterations.last or Iteration() - last_output = last_iteration.validation_output or last_iteration.parsed_output + last_output = last_iteration.validation_response or last_iteration.parsed_output validation_passed = call.status == pass_status reask = last_output if isinstance(last_output, ReAsk) else None error = call.error diff --git a/guardrails/run.py b/guardrails/run.py index 83b6d9bb1..e8c0ef3df 100644 --- a/guardrails/run.py +++ b/guardrails/run.py @@ -183,7 +183,7 @@ def __call__(self, call_log: Call, prompt_params: Optional[Dict] = None) -> Call msg_history, ) = self.prepare_to_loop( iteration.reasks, - call_log.validation_output, + call_log.validation_response, output_schema, prompt_params=prompt_params, include_instructions=include_instructions, @@ -291,7 +291,7 @@ def step( validated_output = self.validate( iteration, index, parsed_output, output_schema ) - iteration.outputs.validation_output = validated_output + iteration.outputs.validation_response = validated_output # Introspect: inspect validated output for reasks. reasks, valid_output = self.introspect( @@ -323,7 +323,7 @@ def validate_msg_history( validated_msg_history = msg_history_schema.validate( iteration, msg_str, self.metadata, disable_tracer=self._disable_tracer ) - iteration.outputs.validation_output = validated_msg_history + iteration.outputs.validation_response = validated_msg_history if isinstance(validated_msg_history, ReAsk): raise ValidationError( f"Message history validation failed: " f"{validated_msg_history}" @@ -366,7 +366,7 @@ def validate_prompt( self.metadata, disable_tracer=self._disable_tracer, ) - iteration.outputs.validation_output = validated_prompt + iteration.outputs.validation_response = validated_prompt if validated_prompt is None: raise ValidationError("Prompt validation failed") if isinstance(validated_prompt, ReAsk): @@ -390,7 +390,7 @@ def validate_instructions( self.metadata, disable_tracer=self._disable_tracer, ) - iteration.outputs.validation_output = validated_instructions + iteration.outputs.validation_response = validated_instructions if validated_instructions is None: raise ValidationError("Instructions validation failed") if isinstance(validated_instructions, ReAsk): @@ -718,7 +718,7 @@ async def async_run( msg_history, ) = self.prepare_to_loop( iteration.reasks, - call_log.validation_output, + call_log.validation_response, output_schema, prompt_params=prompt_params, ) @@ -816,7 +816,7 @@ async def async_step( validated_output = await self.async_validate( iteration, index, parsed_output, output_schema ) - iteration.outputs.validation_output = validated_output + iteration.outputs.validation_response = validated_output # Introspect: inspect validated output for reasks. reasks, valid_output = self.introspect( @@ -975,7 +975,7 @@ async def async_prepare( validated_prompt = await prompt_schema.async_validate( iteration, prompt.source, self.metadata ) - iteration.outputs.validation_output = validated_prompt + iteration.outputs.validation_response = validated_prompt if validated_prompt is None: raise ValidationError("Prompt validation failed") if isinstance(validated_prompt, ReAsk): @@ -994,7 +994,7 @@ async def async_prepare( validated_instructions = await instructions_schema.async_validate( iteration, instructions.source, self.metadata ) - iteration.outputs.validation_output = validated_instructions + iteration.outputs.validation_response = validated_instructions if validated_instructions is None: raise ValidationError("Instructions validation failed") if isinstance(validated_instructions, ReAsk): @@ -1194,7 +1194,7 @@ def step( # Finally, add to logs iteration.outputs.raw_output = fragment iteration.outputs.parsed_output = parsed_fragment - iteration.outputs.validation_output = validated_fragment + iteration.outputs.validation_response = validated_fragment iteration.outputs.validated_output = valid_op def get_chunk_text(self, chunk: Any, api: Union[PromptCallableBase, None]) -> str: diff --git a/guardrails/utils/deprecation.py b/guardrails/utils/deprecation.py new file mode 100644 index 000000000..acba986fa --- /dev/null +++ b/guardrails/utils/deprecation.py @@ -0,0 +1,6 @@ +from string import Template + +deprecation_message = Template( + """'${name}' is deprecated and will be removed in \ +versions ${removal_version} and beyond. Use ${replacement} instead.""" +) diff --git a/tests/integration_tests/test_async.py b/tests/integration_tests/test_async.py index 3873826aa..3037de370 100644 --- a/tests/integration_tests/test_async.py +++ b/tests/integration_tests/test_async.py @@ -64,7 +64,7 @@ async def test_entity_extraction_with_reask(mocker, multiprocessing_validators: assert first.prompt_tokens_consumed == 123 assert first.completion_tokens_consumed == 1234 assert first.raw_output == entity_extraction.LLM_OUTPUT - assert first.validation_output == entity_extraction.VALIDATED_OUTPUT_REASK_1 + assert first.validation_response == entity_extraction.VALIDATED_OUTPUT_REASK_1 # For re-asked prompt and output final = call.iterations.last @@ -109,7 +109,7 @@ async def test_entity_extraction_with_noop(mocker): # For orginal prompt and output assert call.compiled_prompt == entity_extraction.COMPILED_PROMPT assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT - assert call.validation_output == entity_extraction.VALIDATED_OUTPUT_NOOP + assert call.validation_response == entity_extraction.VALIDATED_OUTPUT_NOOP @pytest.mark.asyncio @@ -143,7 +143,7 @@ async def test_entity_extraction_with_noop_pydantic(mocker): # For orginal prompt and output assert call.compiled_prompt == entity_extraction.COMPILED_PROMPT assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT - assert call.validation_output == entity_extraction.VALIDATED_OUTPUT_NOOP + assert call.validation_response == entity_extraction.VALIDATED_OUTPUT_NOOP @pytest.mark.asyncio @@ -176,7 +176,7 @@ async def test_entity_extraction_with_filter(mocker): # For orginal prompt and output assert call.compiled_prompt == entity_extraction.COMPILED_PROMPT assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT - assert call.validation_output == entity_extraction.VALIDATED_OUTPUT_FILTER + assert call.validation_response == entity_extraction.VALIDATED_OUTPUT_FILTER assert call.validated_output is None assert call.status == "fail" diff --git a/tests/integration_tests/test_guard.py b/tests/integration_tests/test_guard.py index 78a85a354..d6246fcfe 100644 --- a/tests/integration_tests/test_guard.py +++ b/tests/integration_tests/test_guard.py @@ -169,7 +169,7 @@ def test_entity_extraction_with_reask( assert first.prompt_tokens_consumed == 123 assert first.completion_tokens_consumed == 1234 assert first.raw_output == entity_extraction.LLM_OUTPUT - assert first.validation_output == entity_extraction.VALIDATED_OUTPUT_REASK_1 + assert first.validation_response == entity_extraction.VALIDATED_OUTPUT_REASK_1 # For reask validator logs two_words_validator_logs = list( @@ -249,7 +249,7 @@ def test_entity_extraction_with_noop(mocker, rail, prompt): assert call.compiled_prompt == entity_extraction.COMPILED_PROMPT assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT assert call.validated_output is None - assert call.validation_output == entity_extraction.VALIDATED_OUTPUT_NOOP + assert call.validation_response == entity_extraction.VALIDATED_OUTPUT_NOOP @pytest.mark.parametrize( @@ -449,7 +449,7 @@ def test_string_reask(mocker): assert call.compiled_instructions == string.COMPILED_INSTRUCTIONS assert call.compiled_prompt == string.COMPILED_PROMPT assert call.iterations.first.raw_output == string.LLM_OUTPUT - assert call.iterations.first.validation_output == string.VALIDATED_OUTPUT_REASK + assert call.iterations.first.validation_response == string.VALIDATED_OUTPUT_REASK # For re-asked prompt and output assert call.iterations.last.inputs.prompt == gd.Prompt(string.COMPILED_PROMPT_REASK) @@ -490,7 +490,7 @@ def test_skeleton_reask(mocker): == entity_extraction.LLM_OUTPUT_SKELETON_REASK_1 ) assert ( - call.iterations.first.validation_output + call.iterations.first.validation_response == entity_extraction.VALIDATED_OUTPUT_SKELETON_REASK_1 ) @@ -609,7 +609,7 @@ def test_entity_extraction_with_reask_with_optional_prompts( assert call.compiled_prompt == expected_prompt assert call.iterations.first.raw_output == entity_extraction.LLM_OUTPUT assert ( - call.iterations.first.validation_output + call.iterations.first.validation_response == entity_extraction.VALIDATED_OUTPUT_REASK_1 ) assert call.compiled_instructions == expected_instructions @@ -670,7 +670,9 @@ def test_string_with_message_history_reask(mocker): assert call.compiled_instructions is None assert call.compiled_prompt is None assert call.iterations.first.raw_output == string.MSG_LLM_OUTPUT_INCORRECT - assert call.iterations.first.validation_output == string.MSG_VALIDATED_OUTPUT_REASK + assert ( + call.iterations.first.validation_response == string.MSG_VALIDATED_OUTPUT_REASK + ) # For re-asked prompt and output assert call.reask_prompts.last == string.MSG_COMPILED_PROMPT_REASK @@ -708,7 +710,7 @@ def test_pydantic_with_message_history_reask(mocker): assert call.compiled_prompt is None assert call.iterations.first.raw_output == pydantic.MSG_HISTORY_LLM_OUTPUT_INCORRECT assert ( - call.iterations.first.validation_output == pydantic.MSG_VALIDATED_OUTPUT_REASK + call.iterations.first.validation_response == pydantic.MSG_VALIDATED_OUTPUT_REASK ) # For re-asked prompt and output @@ -744,7 +746,7 @@ def test_sequential_validator_log_is_not_duplicated(mocker): if x.validator_name == "OneLine" ) assert len(one_line_logs) == len( - guard.history.first.validation_output.get("fees") + guard.history.first.validation_response.get("fees") ) finally: @@ -778,7 +780,7 @@ def test_in_memory_validator_log_is_not_duplicated(mocker): ) assert len(one_line_logs) == len( - guard.history.first.validation_output.get("fees") + guard.history.first.validation_response.get("fees") ) finally: diff --git a/tests/integration_tests/test_multi_reask.py b/tests/integration_tests/test_multi_reask.py index a3597e36f..a75ed52d7 100644 --- a/tests/integration_tests/test_multi_reask.py +++ b/tests/integration_tests/test_multi_reask.py @@ -27,14 +27,14 @@ def test_multi_reask(mocker): assert call.compiled_prompt == python_rail.VALIDATOR_PARALLELISM_PROMPT_1 assert call.raw_outputs.first == python_rail.VALIDATOR_PARALLELISM_RESPONSE_1 assert ( - call.iterations.first.validation_output + call.iterations.first.validation_response == python_rail.VALIDATOR_PARALLELISM_REASK_1 ) assert call.reask_prompts.first == python_rail.VALIDATOR_PARALLELISM_PROMPT_2 assert call.raw_outputs.at(1) == python_rail.VALIDATOR_PARALLELISM_RESPONSE_2 assert ( - call.iterations.at(1).validation_output + call.iterations.at(1).validation_response == python_rail.VALIDATOR_PARALLELISM_REASK_2 ) @@ -42,6 +42,6 @@ def test_multi_reask(mocker): assert call.raw_outputs.last == python_rail.VALIDATOR_PARALLELISM_RESPONSE_3 # The output here fails some validators but passes others. # Since those that it fails in the end are noop fixes, validation fails. - assert call.validation_output == python_rail.VALIDATOR_PARALLELISM_RESPONSE_3 + assert call.validation_response == python_rail.VALIDATOR_PARALLELISM_RESPONSE_3 assert call.validated_output is None assert call.status == "fail" diff --git a/tests/integration_tests/test_pydantic.py b/tests/integration_tests/test_pydantic.py index bb7065894..138230dcd 100644 --- a/tests/integration_tests/test_pydantic.py +++ b/tests/integration_tests/test_pydantic.py @@ -42,7 +42,9 @@ def test_pydantic_with_reask(mocker): # For orginal prompt and output assert call.compiled_prompt == pydantic.COMPILED_PROMPT assert call.iterations.first.raw_output == pydantic.LLM_OUTPUT - assert call.iterations.first.validation_output == pydantic.VALIDATED_OUTPUT_REASK_1 + assert ( + call.iterations.first.validation_response == pydantic.VALIDATED_OUTPUT_REASK_1 + ) # For re-asked prompt and output # Assert through iteration @@ -57,7 +59,7 @@ def test_pydantic_with_reask(mocker): # We don't track merged validation output anymore # Each validation_output is instead tracked as it came back from validation # So this isn't a thing - # assert call.iterations.at(1).validation_output == ( + # assert call.iterations.at(1).validation_response == ( # pydantic.VALIDATED_OUTPUT_REASK_2 # ) @@ -67,7 +69,7 @@ def test_pydantic_with_reask(mocker): ) intermediate_call_state.inputs.full_schema_reask = False assert ( - intermediate_call_state.validation_output == pydantic.VALIDATED_OUTPUT_REASK_2 + intermediate_call_state.validation_response == pydantic.VALIDATED_OUTPUT_REASK_2 ) # For re-asked prompt #2 and output #2 @@ -78,7 +80,7 @@ def test_pydantic_with_reask(mocker): assert call.reask_prompts.last == pydantic.COMPILED_PROMPT_REASK_2 assert call.raw_outputs.last == pydantic.LLM_OUTPUT_REASK_2 assert call.validated_output is None - assert call.validation_output == pydantic.VALIDATED_OUTPUT_REASK_3 + assert call.validation_response == pydantic.VALIDATED_OUTPUT_REASK_3 def test_pydantic_with_full_schema_reask(mocker): @@ -110,7 +112,9 @@ def test_pydantic_with_full_schema_reask(mocker): assert call.compiled_prompt == pydantic.COMPILED_PROMPT_CHAT assert call.compiled_instructions == pydantic.COMPILED_INSTRUCTIONS_CHAT assert call.iterations.first.raw_output == pydantic.LLM_OUTPUT - assert call.iterations.first.validation_output == pydantic.VALIDATED_OUTPUT_REASK_1 + assert ( + call.iterations.first.validation_response == pydantic.VALIDATED_OUTPUT_REASK_1 + ) # For re-asked prompt and output assert call.iterations.at(1).inputs.prompt == gd.Prompt( @@ -120,7 +124,9 @@ def test_pydantic_with_full_schema_reask(mocker): pydantic.COMPILED_INSTRUCTIONS_CHAT ) assert call.iterations.at(1).raw_output == pydantic.LLM_OUTPUT_FULL_REASK_1 - assert call.iterations.at(1).validation_output == pydantic.VALIDATED_OUTPUT_REASK_2 + assert ( + call.iterations.at(1).validation_response == pydantic.VALIDATED_OUTPUT_REASK_2 + ) # For re-asked prompt #2 and output #2 assert call.iterations.last.inputs.prompt == gd.Prompt( @@ -131,7 +137,7 @@ def test_pydantic_with_full_schema_reask(mocker): ) assert call.raw_outputs.last == pydantic.LLM_OUTPUT_FULL_REASK_2 assert call.validated_output is None - assert call.validation_output == pydantic.VALIDATED_OUTPUT_REASK_3 + assert call.validation_response == pydantic.VALIDATED_OUTPUT_REASK_3 class ContainerModel(BaseModel): diff --git a/tests/integration_tests/test_python_rail.py b/tests/integration_tests/test_python_rail.py index f26a0258d..08ec048e0 100644 --- a/tests/integration_tests/test_python_rail.py +++ b/tests/integration_tests/test_python_rail.py @@ -372,7 +372,7 @@ def test_python_string(mocker): assert call.compiled_instructions == string.COMPILED_INSTRUCTIONS assert call.compiled_prompt == string.COMPILED_PROMPT assert call.iterations.first.raw_output == string.LLM_OUTPUT - assert call.iterations.first.validation_output == string.VALIDATED_OUTPUT_REASK + assert call.iterations.first.validation_response == string.VALIDATED_OUTPUT_REASK # For re-asked prompt and output assert call.iterations.last.inputs.prompt == gd.Prompt(string.COMPILED_PROMPT_REASK) diff --git a/tests/unit_tests/classes/history/test_call.py b/tests/unit_tests/classes/history/test_call.py index dce43fa0e..28a1d162c 100644 --- a/tests/unit_tests/classes/history/test_call.py +++ b/tests/unit_tests/classes/history/test_call.py @@ -32,7 +32,7 @@ def test_empty_initialization(): assert call.completion_tokens_consumed is None assert call.raw_outputs == Stack() assert call.parsed_outputs == Stack() - assert call.validation_output is None + assert call.validation_response is None assert call.fixed_output is None assert call.validated_output is None assert call.reasks == Stack() @@ -149,7 +149,7 @@ def custom_llm(): second_outputs = Outputs( llm_response_info=second_llm_response_info, parsed_output=second_parsed_output, - validation_output="Hello there", + validation_response="Hello there", validated_output=second_validated_output, reasks=second_reasks, validator_logs=second_validator_logs, @@ -182,7 +182,7 @@ def custom_llm(): assert call.raw_outputs == Stack("Hello there!", "Hello there") assert call.parsed_outputs == Stack("Hello there!", "Hello there") - assert call.validation_output == "Hello there" + assert call.validation_response == "Hello there" assert call.fixed_output == "Hello there" assert call.validated_output == "Hello there" assert call.reasks == Stack() diff --git a/tests/unit_tests/classes/history/test_iteration.py b/tests/unit_tests/classes/history/test_iteration.py index 7ba6a5920..78d4dcabb 100644 --- a/tests/unit_tests/classes/history/test_iteration.py +++ b/tests/unit_tests/classes/history/test_iteration.py @@ -23,7 +23,7 @@ def test_empty_initialization(): assert iteration.completion_tokens_consumed is None assert iteration.raw_output is None assert iteration.parsed_output is None - assert iteration.validation_output is None + assert iteration.validation_response is None assert iteration.validated_output is None assert iteration.reasks == [] assert iteration.validator_logs == [] @@ -88,7 +88,7 @@ def test_non_empty_initialization(): outputs = Outputs( llm_response_info=llm_response_info, parsed_output=parsed_output, - validation_output=reask, + validation_response=reask, validated_output=validated_output, reasks=reasks, validator_logs=validator_logs, @@ -105,7 +105,7 @@ def test_non_empty_initialization(): assert iteration.completion_tokens_consumed == 3 assert iteration.raw_output == "Hello there!" assert iteration.parsed_output == "Hello there!" - assert iteration.validation_output == reask + assert iteration.validation_response == reask assert iteration.validated_output == "Hello there" assert iteration.reasks == reasks assert iteration.validator_logs == validator_logs diff --git a/tests/unit_tests/classes/history/test_outputs.py b/tests/unit_tests/classes/history/test_outputs.py index c88eca2db..39f800f74 100644 --- a/tests/unit_tests/classes/history/test_outputs.py +++ b/tests/unit_tests/classes/history/test_outputs.py @@ -14,7 +14,7 @@ def test_empty_initialization(): assert empty_outputs.llm_response_info is None assert empty_outputs.parsed_output is None - assert empty_outputs.validation_output is None + assert empty_outputs.validation_response is None assert empty_outputs.validated_output is None assert empty_outputs.reasks == [] assert empty_outputs.validator_logs == [] @@ -190,7 +190,7 @@ def test_status(outputs: Outputs, expected_status: str): ) def test_status_reask(): outputs = Outputs( - validation_output=ReAsk( + validation_response=ReAsk( incorrect_value="Hello there!", fail_results=[non_fixable_fail_result], ), diff --git a/tests/unit_tests/test_validators.py b/tests/unit_tests/test_validators.py index 8aea93410..c4b987598 100644 --- a/tests/unit_tests/test_validators.py +++ b/tests/unit_tests/test_validators.py @@ -709,7 +709,9 @@ def mock_llm_api(*args, **kwargs): mock_llm_api, prompt="What kind of pet should I get?", ) - assert guard.history.first.iterations.first.outputs.validation_output == "What kind" + assert ( + guard.history.first.iterations.first.outputs.validation_response == "What kind" + ) guard = Guard.from_pydantic(output_class=Pet).with_instructions_validation( validators=[TwoWords(on_fail="fix")] ) @@ -719,7 +721,8 @@ def mock_llm_api(*args, **kwargs): instructions="But really, what kind of pet should I get?", ) assert ( - guard.history.first.iterations.first.outputs.validation_output == "But really," + guard.history.first.iterations.first.outputs.validation_response + == "But really," ) # but raises for msg_history validation @@ -758,7 +761,7 @@ def mock_llm_api(*args, **kwargs): guard( mock_llm_api, ) - assert guard.history.first.iterations.first.outputs.validation_output == "This is" + assert guard.history.first.iterations.first.outputs.validation_response == "This is" # rail instructions validation guard = Guard.from_rail_string( @@ -781,7 +784,9 @@ def mock_llm_api(*args, **kwargs): guard( mock_llm_api, ) - assert guard.history.first.iterations.first.outputs.validation_output == "This also" + assert ( + guard.history.first.iterations.first.outputs.validation_response == "This also" + ) @pytest.mark.asyncio @@ -798,7 +803,9 @@ async def mock_llm_api(*args, **kwargs): mock_llm_api, prompt="What kind of pet should I get?", ) - assert guard.history.first.iterations.first.outputs.validation_output == "What kind" + assert ( + guard.history.first.iterations.first.outputs.validation_response == "What kind" + ) guard = Guard.from_pydantic(output_class=Pet).with_instructions_validation( validators=[TwoWords(on_fail="fix")] @@ -809,7 +816,8 @@ async def mock_llm_api(*args, **kwargs): instructions="But really, what kind of pet should I get?", ) assert ( - guard.history.first.iterations.first.outputs.validation_output == "But really," + guard.history.first.iterations.first.outputs.validation_response + == "But really," ) # but raises for msg_history validation @@ -848,7 +856,7 @@ async def mock_llm_api(*args, **kwargs): await guard( mock_llm_api, ) - assert guard.history.first.iterations.first.outputs.validation_output == "This is" + assert guard.history.first.iterations.first.outputs.validation_response == "This is" # rail instructions validation guard = Guard.from_rail_string( @@ -871,7 +879,9 @@ async def mock_llm_api(*args, **kwargs): await guard( mock_llm_api, ) - assert guard.history.first.iterations.first.outputs.validation_output == "This also" + assert ( + guard.history.first.iterations.first.outputs.validation_response == "This also" + ) @pytest.mark.parametrize( From 1007d853e6df98ec3566ed1ffe87ad38b84dd21d Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Thu, 14 Mar 2024 16:00:50 -0500 Subject: [PATCH 2/6] validated_output -> guarded_output --- guardrails/classes/history/call.py | 36 +++++++++++++------ guardrails/classes/history/iteration.py | 15 +++++++- guardrails/classes/history/outputs.py | 17 +++++---- guardrails/classes/validation_outcome.py | 2 +- guardrails/run.py | 6 ++-- guardrails/utils/misc.py | 2 +- tests/integration_tests/test_async.py | 8 ++--- tests/integration_tests/test_guard.py | 22 ++++++------ tests/integration_tests/test_multi_reask.py | 2 +- tests/integration_tests/test_parsing.py | 8 ++--- tests/integration_tests/test_pydantic.py | 4 +-- tests/integration_tests/test_python_rail.py | 2 +- tests/integration_tests/test_run.py | 2 +- tests/unit_tests/classes/history/test_call.py | 4 +-- .../classes/history/test_iteration.py | 8 ++--- .../classes/history/test_outputs.py | 14 ++++---- 16 files changed, 92 insertions(+), 60 deletions(-) diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py index 2bad3e7ec..c15c1dcd7 100644 --- a/guardrails/classes/history/call.py +++ b/guardrails/classes/history/call.py @@ -226,27 +226,29 @@ def validation_response(self) -> Optional[Union[str, Dict, ReAsk]]: if ( self.inputs.full_schema_reask or number_of_iterations < 2 - or isinstance(self.iterations.last.validation_output, ReAsk) # type: ignore - or isinstance(self.iterations.last.validation_output, str) # type: ignore + or isinstance( + self.iterations.last.validation_response, ReAsk # type: ignore + ) + or isinstance(self.iterations.last.validation_response, str) # type: ignore ): - return self.iterations.last.validation_output # type: ignore + return self.iterations.last.validation_response # type: ignore current_index = 1 # We've already established that there are iterations, # hence the type ignores - merged_validation_output = ( - self.iterations.first.validation_output # type: ignore + merged_validation_responses = ( + self.iterations.first.validation_response # type: ignore ) while current_index < number_of_iterations: current_validation_output = self.iterations.at( current_index - ).validation_output # type: ignore - merged_validation_output = merge_reask_output( - merged_validation_output, current_validation_output + ).validation_response # type: ignore + merged_validation_responses = merge_reask_output( + merged_validation_responses, current_validation_output ) current_index = current_index + 1 - return merged_validation_output + return merged_validation_responses @property def fixed_output(self) -> Optional[Union[str, Dict]]: @@ -258,7 +260,7 @@ def fixed_output(self) -> Optional[Union[str, Dict]]: return sub_reasks_with_fixed_values(self.validation_response) @property - def validated_output(self) -> Optional[Union[str, Dict]]: + def guarded_output(self) -> Optional[Union[str, Dict]]: """The output from the LLM after undergoing validation. This will only have a value if the Guard is in a passing state. @@ -266,6 +268,18 @@ def validated_output(self) -> Optional[Union[str, Dict]]: if self.status == pass_status: return self.fixed_output + @property + @deprecated( + """'Call.validated_output' is deprecated and will be removed in \ +versions 0.5.0 and beyond. Use 'guarded_output' instead.""" + ) + def validated_output(self) -> Optional[Union[str, Dict]]: + """The output from the LLM after undergoing validation. + + This will only have a value if the Guard is in a passing state. + """ + return self.guarded_output + @property def reasks(self) -> Stack[ReAsk]: """Reasks generated during validation that could not be automatically @@ -368,7 +382,7 @@ def tree(self) -> Tree: :-1 ] validated_outcome_panel = Panel( - pretty_repr(self.validated_output), + pretty_repr(self.guarded_output), title="Validated Output", style="on #F0FFF0", ) diff --git a/guardrails/classes/history/iteration.py b/guardrails/classes/history/iteration.py index bc0717eb4..31ff3004e 100644 --- a/guardrails/classes/history/iteration.py +++ b/guardrails/classes/history/iteration.py @@ -97,13 +97,26 @@ def validation_output(self) -> Optional[Union[ReAsk, str, Dict]]: return self.validation_response @property + def guarded_output(self) -> Optional[Union[str, Dict]]: + """The valid output from the LLM after undergoing validation. + + Could be only a partial structure if field level reasks occur. + Could contain fixed values. + """ + return self.outputs.guarded_output + + @property + @deprecated( + """'Iteration.validated_output' is deprecated and will be removed in \ +versions 0.5.0 and beyond. Use 'guarded_output' instead.""" + ) def validated_output(self) -> Optional[Union[str, Dict]]: """The valid output from the LLM after undergoing validation. Could be only a partial structure if field level reasks occur. Could contain fixed values. """ - return self.outputs.validated_output + return self.outputs.guarded_output @property def reasks(self) -> Sequence[ReAsk]: diff --git a/guardrails/classes/history/outputs.py b/guardrails/classes/history/outputs.py index e76be4032..7a239b8d9 100644 --- a/guardrails/classes/history/outputs.py +++ b/guardrails/classes/history/outputs.py @@ -26,10 +26,7 @@ class Outputs(ArbitraryModel): validation_response: Optional[Union[str, ReAsk, Dict]] = Field( description="The response from the validation process.", default=None ) - # validation_output: Optional[Union[str, ReAsk, Dict]] = Field( - # description="The output from the validation process.", default=None - # ) - validated_output: Optional[Union[str, Dict]] = Field( + guarded_output: Optional[Union[str, Dict]] = Field( description="The valid output after validation." "Could be only a partial structure if field level reasks occur." "Could contain fixed values.", @@ -59,7 +56,7 @@ def _all_empty(self) -> bool: self.llm_response_info is None and self.parsed_output is None and self.validation_response is None - and self.validated_output is None + and self.guarded_output is None and len(self.reasks) == 0 and len(self.validator_logs) == 0 and self.error is None @@ -97,7 +94,7 @@ def status(self) -> str: return error_status elif not all_reasks_have_fixes: return fail_status - elif self.validated_output is None and isinstance( + elif self.guarded_output is None and isinstance( self.validation_response, ReAsk ): return fail_status @@ -110,3 +107,11 @@ def status(self) -> str: ) def validation_output(self) -> Optional[Union[str, ReAsk, Dict]]: return self.validation_response + + @property + @deprecated( + """'Outputs.validated_output' is deprecated and will be removed in \ +versions 0.5.0 and beyond. Use 'guarded_output' instead.""" + ) + def validated_output(self) -> Optional[Union[str, ReAsk, Dict]]: + return self.guarded_output diff --git a/guardrails/classes/validation_outcome.py b/guardrails/classes/validation_outcome.py index fae18fe90..c59eadf2d 100644 --- a/guardrails/classes/validation_outcome.py +++ b/guardrails/classes/validation_outcome.py @@ -55,7 +55,7 @@ def from_guard_history(cls, call: Call): validation_passed = call.status == pass_status reask = last_output if isinstance(last_output, ReAsk) else None error = call.error - output = cast(OT, call.validated_output) + output = cast(OT, call.guarded_output) return cls( raw_llm_output=call.raw_outputs.last, validated_output=output, diff --git a/guardrails/run.py b/guardrails/run.py index e8c0ef3df..055d38696 100644 --- a/guardrails/run.py +++ b/guardrails/run.py @@ -297,7 +297,7 @@ def step( reasks, valid_output = self.introspect( index, validated_output, output_schema ) - iteration.outputs.validated_output = valid_output + iteration.outputs.guarded_output = valid_output iteration.outputs.reasks = reasks @@ -822,7 +822,7 @@ async def async_step( reasks, valid_output = self.introspect( index, validated_output, output_schema ) - iteration.outputs.validated_output = valid_output + iteration.outputs.guarded_output = valid_output iteration.outputs.reasks = reasks @@ -1195,7 +1195,7 @@ def step( iteration.outputs.raw_output = fragment iteration.outputs.parsed_output = parsed_fragment iteration.outputs.validation_response = validated_fragment - iteration.outputs.validated_output = valid_op + iteration.outputs.guarded_output = valid_op def get_chunk_text(self, chunk: Any, api: Union[PromptCallableBase, None]) -> str: """Get the text from a chunk.""" diff --git a/guardrails/utils/misc.py b/guardrails/utils/misc.py index db2cff8fe..fa22137b7 100644 --- a/guardrails/utils/misc.py +++ b/guardrails/utils/misc.py @@ -63,7 +63,7 @@ def generate_test_artifacts( f.write(llm_output or "") # Save the validated response. - validated_output = logs.validated_output + validated_output = logs.guarded_output with open( os.path.join(artifact_dir, f"validated_response_{on_fail_type}{ext}.py"), "w", diff --git a/tests/integration_tests/test_async.py b/tests/integration_tests/test_async.py index 3037de370..8eb8ff8cf 100644 --- a/tests/integration_tests/test_async.py +++ b/tests/integration_tests/test_async.py @@ -72,7 +72,7 @@ async def test_entity_extraction_with_reask(mocker, multiprocessing_validators: # Same as above assert call.reask_prompts.last == entity_extraction.COMPILED_PROMPT_REASK assert final.raw_output == entity_extraction.LLM_OUTPUT_REASK - assert call.validated_output == entity_extraction.VALIDATED_OUTPUT_REASK_2 + assert call.guarded_output == entity_extraction.VALIDATED_OUTPUT_REASK_2 @pytest.mark.asyncio @@ -177,7 +177,7 @@ async def test_entity_extraction_with_filter(mocker): assert call.compiled_prompt == entity_extraction.COMPILED_PROMPT assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT assert call.validation_response == entity_extraction.VALIDATED_OUTPUT_FILTER - assert call.validated_output is None + assert call.guarded_output is None assert call.status == "fail" @@ -210,7 +210,7 @@ async def test_entity_extraction_with_fix(mocker): # For orginal prompt and output assert call.compiled_prompt == entity_extraction.COMPILED_PROMPT assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT - assert call.validated_output == entity_extraction.VALIDATED_OUTPUT_FIX + assert call.guarded_output == entity_extraction.VALIDATED_OUTPUT_FIX @pytest.mark.asyncio @@ -242,7 +242,7 @@ async def test_entity_extraction_with_refrain(mocker): # For orginal prompt and output assert call.compiled_prompt == entity_extraction.COMPILED_PROMPT assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT - assert call.validated_output == entity_extraction.VALIDATED_OUTPUT_REFRAIN + assert call.guarded_output == entity_extraction.VALIDATED_OUTPUT_REFRAIN @pytest.mark.asyncio diff --git a/tests/integration_tests/test_guard.py b/tests/integration_tests/test_guard.py index d6246fcfe..d963e2fce 100644 --- a/tests/integration_tests/test_guard.py +++ b/tests/integration_tests/test_guard.py @@ -214,7 +214,7 @@ def test_entity_extraction_with_reask( # Second iteration is the first reask assert call.reask_prompts.first == entity_extraction.COMPILED_PROMPT_REASK assert call.raw_outputs.at(1) == entity_extraction.LLM_OUTPUT_REASK - assert call.validated_output == entity_extraction.VALIDATED_OUTPUT_REASK_2 + assert call.guarded_output == entity_extraction.VALIDATED_OUTPUT_REASK_2 @pytest.mark.parametrize( @@ -248,7 +248,7 @@ def test_entity_extraction_with_noop(mocker, rail, prompt): # For orginal prompt and output assert call.compiled_prompt == entity_extraction.COMPILED_PROMPT assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT - assert call.validated_output is None + assert call.guarded_output is None assert call.validation_response == entity_extraction.VALIDATED_OUTPUT_NOOP @@ -287,7 +287,7 @@ def test_entity_extraction_with_filter(mocker, rail, prompt): assert call.compiled_prompt == entity_extraction.COMPILED_PROMPT assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT assert call.status == "fail" - assert call.validated_output is None + assert call.guarded_output is None @pytest.mark.parametrize( @@ -320,7 +320,7 @@ def test_entity_extraction_with_fix(mocker, rail, prompt): # For orginal prompt and output assert call.compiled_prompt == entity_extraction.COMPILED_PROMPT assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT - assert call.validated_output == entity_extraction.VALIDATED_OUTPUT_FIX + assert call.guarded_output == entity_extraction.VALIDATED_OUTPUT_FIX @pytest.mark.parametrize( @@ -356,7 +356,7 @@ def test_entity_extraction_with_refrain(mocker, rail, prompt): # For orginal prompt and output assert call.compiled_prompt == entity_extraction.COMPILED_PROMPT assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT - assert call.validated_output == entity_extraction.VALIDATED_OUTPUT_REFRAIN + assert call.guarded_output == entity_extraction.VALIDATED_OUTPUT_REFRAIN @pytest.mark.parametrize( @@ -400,7 +400,7 @@ def test_entity_extraction_with_fix_chat_models(mocker, rail, prompt, instructio ) assert call.compiled_instructions == entity_extraction.COMPILED_INSTRUCTIONS assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT - assert call.validated_output == entity_extraction.VALIDATED_OUTPUT_FIX + assert call.guarded_output == entity_extraction.VALIDATED_OUTPUT_FIX def test_string_output(mocker): @@ -457,7 +457,7 @@ def test_string_reask(mocker): assert call.reask_prompts.last == string.COMPILED_PROMPT_REASK assert call.raw_outputs.last == string.LLM_OUTPUT_REASK - assert call.validated_output == string.LLM_OUTPUT_REASK + assert call.guarded_output == string.LLM_OUTPUT_REASK def test_skeleton_reask(mocker): @@ -497,7 +497,7 @@ def test_skeleton_reask(mocker): # For re-asked prompt and output assert call.reask_prompts.last == entity_extraction.COMPILED_PROMPT_SKELETON_REASK_2 assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT_SKELETON_REASK_2 - assert call.validated_output == entity_extraction.VALIDATED_OUTPUT_SKELETON_REASK_2 + assert call.guarded_output == entity_extraction.VALIDATED_OUTPUT_SKELETON_REASK_2 '''def test_json_output(mocker): @@ -639,7 +639,7 @@ def test_entity_extraction_with_reask_with_optional_prompts( assert call.reask_prompts.last == expected_reask_prompt assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT_REASK - assert call.validated_output == entity_extraction.VALIDATED_OUTPUT_REASK_2 + assert call.guarded_output == entity_extraction.VALIDATED_OUTPUT_REASK_2 if expected_reask_instructions: assert call.reask_instructions.last == expected_reask_instructions @@ -678,7 +678,7 @@ def test_string_with_message_history_reask(mocker): assert call.reask_prompts.last == string.MSG_COMPILED_PROMPT_REASK assert call.reask_instructions.last == string.MSG_COMPILED_INSTRUCTIONS_REASK assert call.raw_outputs.last == string.MSG_LLM_OUTPUT_CORRECT - assert call.validated_output == string.MSG_LLM_OUTPUT_CORRECT + assert call.guarded_output == string.MSG_LLM_OUTPUT_CORRECT def test_pydantic_with_message_history_reask(mocker): @@ -717,7 +717,7 @@ def test_pydantic_with_message_history_reask(mocker): assert call.reask_prompts.last == pydantic.MSG_COMPILED_PROMPT_REASK assert call.reask_instructions.last == pydantic.MSG_COMPILED_INSTRUCTIONS_REASK assert call.raw_outputs.last == pydantic.MSG_HISTORY_LLM_OUTPUT_CORRECT - assert call.validated_output == json.loads(pydantic.MSG_HISTORY_LLM_OUTPUT_CORRECT) + assert call.guarded_output == json.loads(pydantic.MSG_HISTORY_LLM_OUTPUT_CORRECT) def test_sequential_validator_log_is_not_duplicated(mocker): diff --git a/tests/integration_tests/test_multi_reask.py b/tests/integration_tests/test_multi_reask.py index a75ed52d7..4dd8ade51 100644 --- a/tests/integration_tests/test_multi_reask.py +++ b/tests/integration_tests/test_multi_reask.py @@ -43,5 +43,5 @@ def test_multi_reask(mocker): # The output here fails some validators but passes others. # Since those that it fails in the end are noop fixes, validation fails. assert call.validation_response == python_rail.VALIDATOR_PARALLELISM_RESPONSE_3 - assert call.validated_output is None + assert call.guarded_output is None assert call.status == "fail" diff --git a/tests/integration_tests/test_parsing.py b/tests/integration_tests/test_parsing.py index fe6c6e044..ba72b9651 100644 --- a/tests/integration_tests/test_parsing.py +++ b/tests/integration_tests/test_parsing.py @@ -44,7 +44,7 @@ def mock_callable(prompt: str): # For orginal prompt and output assert call.compiled_prompt == pydantic.PARSING_COMPILED_PROMPT assert call.iterations.first.raw_output == pydantic.PARSING_UNPARSEABLE_LLM_OUTPUT - assert call.iterations.first.validated_output is None + assert call.iterations.first.guarded_output is None # For re-asked prompt and output assert call.iterations.last.inputs.prompt == gd.Prompt( @@ -53,7 +53,7 @@ def mock_callable(prompt: str): # Same as above assert call.reask_prompts.last == pydantic.PARSING_COMPILED_REASK assert call.raw_outputs.last == pydantic.PARSING_EXPECTED_LLM_OUTPUT - assert call.validated_output == pydantic.PARSING_EXPECTED_OUTPUT + assert call.guarded_output == pydantic.PARSING_EXPECTED_OUTPUT @pytest.mark.asyncio @@ -87,7 +87,7 @@ async def mock_async_callable(prompt: str): # For orginal prompt and output assert call.compiled_prompt == pydantic.PARSING_COMPILED_PROMPT assert call.iterations.first.raw_output == pydantic.PARSING_UNPARSEABLE_LLM_OUTPUT - assert call.iterations.first.validated_output is None + assert call.iterations.first.guarded_output is None # For re-asked prompt and output assert call.iterations.last.inputs.prompt == gd.Prompt( @@ -96,7 +96,7 @@ async def mock_async_callable(prompt: str): # Same as above assert call.reask_prompts.last == pydantic.PARSING_COMPILED_REASK assert call.raw_outputs.last == pydantic.PARSING_EXPECTED_LLM_OUTPUT - assert call.validated_output == pydantic.PARSING_EXPECTED_OUTPUT + assert call.guarded_output == pydantic.PARSING_EXPECTED_OUTPUT def test_reask_prompt_instructions(mocker): diff --git a/tests/integration_tests/test_pydantic.py b/tests/integration_tests/test_pydantic.py index 138230dcd..67e5c72a4 100644 --- a/tests/integration_tests/test_pydantic.py +++ b/tests/integration_tests/test_pydantic.py @@ -79,7 +79,7 @@ def test_pydantic_with_reask(mocker): # Same as above assert call.reask_prompts.last == pydantic.COMPILED_PROMPT_REASK_2 assert call.raw_outputs.last == pydantic.LLM_OUTPUT_REASK_2 - assert call.validated_output is None + assert call.guarded_output is None assert call.validation_response == pydantic.VALIDATED_OUTPUT_REASK_3 @@ -136,7 +136,7 @@ def test_pydantic_with_full_schema_reask(mocker): pydantic.COMPILED_INSTRUCTIONS_CHAT ) assert call.raw_outputs.last == pydantic.LLM_OUTPUT_FULL_REASK_2 - assert call.validated_output is None + assert call.guarded_output is None assert call.validation_response == pydantic.VALIDATED_OUTPUT_REASK_3 diff --git a/tests/integration_tests/test_python_rail.py b/tests/integration_tests/test_python_rail.py index 08ec048e0..04db07a48 100644 --- a/tests/integration_tests/test_python_rail.py +++ b/tests/integration_tests/test_python_rail.py @@ -379,4 +379,4 @@ def test_python_string(mocker): # Same as above assert call.reask_prompts.last == string.COMPILED_PROMPT_REASK assert call.raw_outputs.last == string.LLM_OUTPUT_REASK - assert call.validated_output == string.LLM_OUTPUT_REASK + assert call.guarded_output == string.LLM_OUTPUT_REASK diff --git a/tests/integration_tests/test_run.py b/tests/integration_tests/test_run.py index d24acd358..49496928c 100644 --- a/tests/integration_tests/test_run.py +++ b/tests/integration_tests/test_run.py @@ -159,5 +159,5 @@ async def test_sync_async_step_equivalence(mocker): OUTPUT, ) - assert sync_iteration.validated_output == async_iteration.validated_output + assert sync_iteration.guarded_output == async_iteration.guarded_output assert sync_iteration.reasks == async_iteration.reasks diff --git a/tests/unit_tests/classes/history/test_call.py b/tests/unit_tests/classes/history/test_call.py index 28a1d162c..dca013268 100644 --- a/tests/unit_tests/classes/history/test_call.py +++ b/tests/unit_tests/classes/history/test_call.py @@ -34,7 +34,7 @@ def test_empty_initialization(): assert call.parsed_outputs == Stack() assert call.validation_response is None assert call.fixed_output is None - assert call.validated_output is None + assert call.guarded_output is None assert call.reasks == Stack() assert call.validator_logs == Stack() assert call.error is None @@ -184,7 +184,7 @@ def custom_llm(): assert call.parsed_outputs == Stack("Hello there!", "Hello there") assert call.validation_response == "Hello there" assert call.fixed_output == "Hello there" - assert call.validated_output == "Hello there" + assert call.guarded_output == "Hello there" assert call.reasks == Stack() assert call.validator_logs == Stack(first_validator_log, second_validator_log) assert call.error is None diff --git a/tests/unit_tests/classes/history/test_iteration.py b/tests/unit_tests/classes/history/test_iteration.py index 78d4dcabb..8c4ac17d2 100644 --- a/tests/unit_tests/classes/history/test_iteration.py +++ b/tests/unit_tests/classes/history/test_iteration.py @@ -24,7 +24,7 @@ def test_empty_initialization(): assert iteration.raw_output is None assert iteration.parsed_output is None assert iteration.validation_response is None - assert iteration.validated_output is None + assert iteration.guarded_output is None assert iteration.reasks == [] assert iteration.validator_logs == [] assert iteration.error is None @@ -69,7 +69,7 @@ def test_non_empty_initialization(): output="Hello there!", prompt_token_count=10, response_token_count=3 ) parsed_output = "Hello there!" - validated_output = "Hello there" + guarded_output = "Hello there" reask = FieldReAsk( incorrect_value="Hello there!", fail_results=[validation_result], path=[] ) @@ -89,7 +89,7 @@ def test_non_empty_initialization(): llm_response_info=llm_response_info, parsed_output=parsed_output, validation_response=reask, - validated_output=validated_output, + guarded_output=guarded_output, reasks=reasks, validator_logs=validator_logs, error=error, @@ -106,7 +106,7 @@ def test_non_empty_initialization(): assert iteration.raw_output == "Hello there!" assert iteration.parsed_output == "Hello there!" assert iteration.validation_response == reask - assert iteration.validated_output == "Hello there" + assert iteration.guarded_output == "Hello there" assert iteration.reasks == reasks assert iteration.validator_logs == validator_logs assert iteration.error == error diff --git a/tests/unit_tests/classes/history/test_outputs.py b/tests/unit_tests/classes/history/test_outputs.py index 39f800f74..e967432cf 100644 --- a/tests/unit_tests/classes/history/test_outputs.py +++ b/tests/unit_tests/classes/history/test_outputs.py @@ -15,7 +15,7 @@ def test_empty_initialization(): assert empty_outputs.llm_response_info is None assert empty_outputs.parsed_output is None assert empty_outputs.validation_response is None - assert empty_outputs.validated_output is None + assert empty_outputs.guarded_output is None assert empty_outputs.reasks == [] assert empty_outputs.validator_logs == [] assert empty_outputs.error is None @@ -33,7 +33,7 @@ def test_non_empty_initialization(): output="Hello there!", prompt_token_count=10, response_token_count=3 ) parsed_output = "Hello there!" - validated_output = "Hello there" + guarded_output = "Hello there" reasks = [ReAsk(incorrect_value="Hello there!", fail_results=[validation_result])] validator_logs = [ ValidatorLogs( @@ -49,7 +49,7 @@ def test_non_empty_initialization(): non_empty_outputs = Outputs( llm_response_info=llm_response_info, parsed_output=parsed_output, - validated_output=validated_output, + guarded_output=guarded_output, reasks=reasks, validator_logs=validator_logs, error=error, @@ -59,8 +59,8 @@ def test_non_empty_initialization(): assert non_empty_outputs.llm_response_info == llm_response_info assert non_empty_outputs.parsed_output is not None assert non_empty_outputs.parsed_output == parsed_output - assert non_empty_outputs.validated_output is not None - assert non_empty_outputs.validated_output == validated_output + assert non_empty_outputs.guarded_output is not None + assert non_empty_outputs.guarded_output == guarded_output assert non_empty_outputs.reasks != [] assert non_empty_outputs.reasks == reasks assert non_empty_outputs.validator_logs != [] @@ -89,7 +89,7 @@ def test_non_empty_initialization(): (Outputs(llm_response_info=LLMResponse(output="Hello there!")), False), (Outputs(parsed_output="Hello there!"), False), (Outputs(parsed_output="Hello there!"), False), - (Outputs(validated_output="Hello there"), False), + (Outputs(guarded_output="Hello there"), False), ( Outputs( reasks=[ @@ -175,7 +175,7 @@ def test_failed_validations(): ), fail_status, ), - (Outputs(validator_logs=[], validated_output="Hello there!"), pass_status), + (Outputs(validator_logs=[], guarded_output="Hello there!"), pass_status), ], ) def test_status(outputs: Outputs, expected_status: str): From 57967ee74d2757fe0e16f4411d9a22a615b9cf3b Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Thu, 14 Mar 2024 16:08:22 -0500 Subject: [PATCH 3/6] docstring, remove unused --- guardrails/classes/history/call.py | 2 +- guardrails/utils/deprecation.py | 6 ------ 2 files changed, 1 insertion(+), 7 deletions(-) delete mode 100644 guardrails/utils/deprecation.py diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py index c15c1dcd7..325da3cdb 100644 --- a/guardrails/classes/history/call.py +++ b/guardrails/classes/history/call.py @@ -209,7 +209,7 @@ def validation_output(self) -> Optional[Union[str, Dict, ReAsk]]: @property def validation_response(self) -> Optional[Union[str, Dict, ReAsk]]: """The aggregated responses from the validation process across all - iterations with the current call. + iterations within the current call. Could contain ReAsks. """ diff --git a/guardrails/utils/deprecation.py b/guardrails/utils/deprecation.py deleted file mode 100644 index acba986fa..000000000 --- a/guardrails/utils/deprecation.py +++ /dev/null @@ -1,6 +0,0 @@ -from string import Template - -deprecation_message = Template( - """'${name}' is deprecated and will be removed in \ -versions ${removal_version} and beyond. Use ${replacement} instead.""" -) From 68eaaec593b3cee816f3b188756b744b543828d5 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Fri, 15 Mar 2024 08:18:54 -0500 Subject: [PATCH 4/6] tweak doc strings --- guardrails/classes/history/call.py | 2 +- guardrails/classes/history/iteration.py | 6 +++--- guardrails/classes/history/outputs.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py index 325da3cdb..ba2511df0 100644 --- a/guardrails/classes/history/call.py +++ b/guardrails/classes/history/call.py @@ -261,7 +261,7 @@ def fixed_output(self) -> Optional[Union[str, Dict]]: @property def guarded_output(self) -> Optional[Union[str, Dict]]: - """The output from the LLM after undergoing validation. + """The final output after undergoing validation. This will only have a value if the Guard is in a passing state. """ diff --git a/guardrails/classes/history/iteration.py b/guardrails/classes/history/iteration.py index 31ff3004e..d9160e245 100644 --- a/guardrails/classes/history/iteration.py +++ b/guardrails/classes/history/iteration.py @@ -80,7 +80,7 @@ def parsed_output(self) -> Optional[Union[str, Dict]]: def validation_response(self) -> Optional[Union[ReAsk, str, Dict]]: """The response from the validation process. - Could be a combination of valid output and ReAsks + Could be a combination of valid output and ReAsks. """ return self.outputs.validation_response @@ -98,9 +98,9 @@ def validation_output(self) -> Optional[Union[ReAsk, str, Dict]]: @property def guarded_output(self) -> Optional[Union[str, Dict]]: - """The valid output from the LLM after undergoing validation. + """Any valid output after undergoing validation. - Could be only a partial structure if field level reasks occur. + Could be a partial structure if field level reasks occur. Could contain fixed values. """ return self.outputs.guarded_output diff --git a/guardrails/classes/history/outputs.py b/guardrails/classes/history/outputs.py index 7a239b8d9..6eb1f4a98 100644 --- a/guardrails/classes/history/outputs.py +++ b/guardrails/classes/history/outputs.py @@ -27,8 +27,8 @@ class Outputs(ArbitraryModel): description="The response from the validation process.", default=None ) guarded_output: Optional[Union[str, Dict]] = Field( - description="The valid output after validation." - "Could be only a partial structure if field level reasks occur." + description="Any valid output after validation." + "Could be a partial structure if field level reasks occur." "Could contain fixed values.", default=None, ) From 82129fa5421ac4f227a1a4d808924d62f8664590 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Fri, 15 Mar 2024 14:58:14 -0500 Subject: [PATCH 5/6] update iteration.validation_response docstring --- guardrails/classes/history/iteration.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/guardrails/classes/history/iteration.py b/guardrails/classes/history/iteration.py index d9160e245..b36a25c91 100644 --- a/guardrails/classes/history/iteration.py +++ b/guardrails/classes/history/iteration.py @@ -78,9 +78,13 @@ def parsed_output(self) -> Optional[Union[str, Dict]]: @property def validation_response(self) -> Optional[Union[ReAsk, str, Dict]]: - """The response from the validation process. + """The response from a single stage of validation. - Could be a combination of valid output and ReAsks. + Validation response is the output of a single stage of validation + and could be a combination of valid output and reasks. + Note that a Guard may run validation multiple times if reasks occur. + To access the final output after all steps of validation are completed, + check out `Call.guarded_output`." """ return self.outputs.validation_response From 89be7c7c954ad82caa4f5b4ef815e858e5399750 Mon Sep 17 00:00:00 2001 From: Caleb Courier Date: Fri, 15 Mar 2024 15:05:15 -0500 Subject: [PATCH 6/6] update doc strings --- guardrails/classes/history/call.py | 10 ++++++++-- guardrails/classes/history/iteration.py | 7 ++++--- guardrails/classes/history/outputs.py | 7 ++++--- 3 files changed, 16 insertions(+), 8 deletions(-) diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py index ba2511df0..98b5a7971 100644 --- a/guardrails/classes/history/call.py +++ b/guardrails/classes/history/call.py @@ -211,7 +211,7 @@ def validation_response(self) -> Optional[Union[str, Dict, ReAsk]]: """The aggregated responses from the validation process across all iterations within the current call. - Could contain ReAsks. + This value could contain ReAsks. """ number_of_iterations = self.iterations.length @@ -261,7 +261,13 @@ def fixed_output(self) -> Optional[Union[str, Dict]]: @property def guarded_output(self) -> Optional[Union[str, Dict]]: - """The final output after undergoing validation. + """The complete validated output after all stages of validation are + completed. + + This property contains the aggregate validated output after all + validation stages have been completed. Some values in the + validated output may be "fixed" values that were corrected + during validation. This will only have a value if the Guard is in a passing state. """ diff --git a/guardrails/classes/history/iteration.py b/guardrails/classes/history/iteration.py index b36a25c91..7d96b56ea 100644 --- a/guardrails/classes/history/iteration.py +++ b/guardrails/classes/history/iteration.py @@ -102,10 +102,11 @@ def validation_output(self) -> Optional[Union[ReAsk, str, Dict]]: @property def guarded_output(self) -> Optional[Union[str, Dict]]: - """Any valid output after undergoing validation. + """Any valid values after undergoing validation. - Could be a partial structure if field level reasks occur. - Could contain fixed values. + Some values in the validated output may be "fixed" values that + were corrected during validation. This property may be a partial + structure if field level reasks occur. """ return self.outputs.guarded_output diff --git a/guardrails/classes/history/outputs.py b/guardrails/classes/history/outputs.py index 6eb1f4a98..9d4d19544 100644 --- a/guardrails/classes/history/outputs.py +++ b/guardrails/classes/history/outputs.py @@ -27,9 +27,10 @@ class Outputs(ArbitraryModel): description="The response from the validation process.", default=None ) guarded_output: Optional[Union[str, Dict]] = Field( - description="Any valid output after validation." - "Could be a partial structure if field level reasks occur." - "Could contain fixed values.", + description="""Any valid values after undergoing validation. + + Some values may be "fixed" values that were corrected during validation. + This property may be a partial structure if field level reasks occur.""", default=None, ) reasks: Sequence[ReAsk] = Field(