diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd80b0da5..fcd78e6c3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -173,7 +173,7 @@ jobs: - name: Install dependencies run: | make full; - poetry add openai==0.28.1 jupyter nbconvert cohere; + poetry add "openai>=1.2.4" jupyter nbconvert cohere; - name: Check for pypdfium2 run: poetry run pip show pypdfium2 - name: Huggingface Hub Login diff --git a/.github/workflows/examples_check.yml b/.github/workflows/examples_check.yml index bbbb2ced1..d700dc3cf 100644 --- a/.github/workflows/examples_check.yml +++ b/.github/workflows/examples_check.yml @@ -38,7 +38,7 @@ jobs: - name: Install dependencies run: | make full; - poetry add openai==0.28.1 jupyter nbconvert cohere; + poetry add "openai>=1.2.4" jupyter nbconvert cohere; - name: Check for pypdfium2 run: poetry run pip show pypdfium2 - name: Huggingface Hub Login @@ -48,7 +48,7 @@ jobs: mkdir /tmp/nltk_data; poetry run python -m nltk.downloader -d /tmp/nltk_data punkt; - name: Use venv - run: source .venv/bin/activate + run: source $VENV - name: Execute notebooks and check for errors run: bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }} diff --git a/docs/examples/extracting_entities.ipynb b/docs/examples/extracting_entities.ipynb index 5b53ac222..2decaec8f 100644 --- a/docs/examples/extracting_entities.ipynb +++ b/docs/examples/extracting_entities.ipynb @@ -342,7 +342,7 @@ "source": [ "import openai\n", "\n", - "validated_response = guard(\n", + "_, validated_response, *rest = guard(\n", " openai.chat.completions.create,\n", " prompt_params={\"document\": content[:6000]},\n", " max_tokens=2048,\n", @@ -1724,7 +1724,7 @@ } ], "source": [ - "guard.state.most_recent_call.tree" + "guard.history.last.tree" ] } ], diff --git a/docs/examples/generate_structured_data.ipynb b/docs/examples/generate_structured_data.ipynb index acdb40345..e5fa6036e 100644 --- a/docs/examples/generate_structured_data.ipynb +++ b/docs/examples/generate_structured_data.ipynb @@ -279,7 +279,7 @@ "import openai\n", "\n", "\n", - "validated_output = guard(\n", + "_, validated_output, *rest = guard(\n", " openai.chat.completions.create,\n", " max_tokens=2048,\n", " temperature=0\n", diff --git a/docs/examples/guardrails_with_chat_models.ipynb b/docs/examples/guardrails_with_chat_models.ipynb index 91b7bfb21..ea47425a5 100644 --- a/docs/examples/guardrails_with_chat_models.ipynb +++ b/docs/examples/guardrails_with_chat_models.ipynb @@ -1712,7 +1712,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.17" + "version": "3.11.6" }, "orig_nbformat": 4 }, diff --git a/docs/examples/input_validation.ipynb b/docs/examples/input_validation.ipynb index 28853a40c..d3e23fbb7 100644 --- a/docs/examples/input_validation.ipynb +++ b/docs/examples/input_validation.ipynb @@ -136,7 +136,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.17" + "version": "3.11.6" } }, "nbformat": 4, diff --git a/docs/examples/no_secrets_in_generated_text.ipynb b/docs/examples/no_secrets_in_generated_text.ipynb index 6d540214a..65a97b56d 100644 --- a/docs/examples/no_secrets_in_generated_text.ipynb +++ b/docs/examples/no_secrets_in_generated_text.ipynb @@ -293,6 +293,19 @@ "We can see that the output is a dictionary with the correct schema and types." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import openai\n", + "\n", + "raw_llm_response, validated_response, *rest = guard(\n", + " openai.completions.create, model=\"text-davinci-003\", max_tokens=2048, temperature=0\n", + ")" + ] + }, { "cell_type": "code", "execution_count": 9, diff --git a/docs/examples/response_is_on_topic.ipynb b/docs/examples/response_is_on_topic.ipynb index 6d736e0f8..d876108fc 100644 --- a/docs/examples/response_is_on_topic.ipynb +++ b/docs/examples/response_is_on_topic.ipynb @@ -279,7 +279,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.17" + "version": "3.11.6" } }, "nbformat": 4, diff --git a/docs/examples/select_choice_based_on_action.ipynb b/docs/examples/select_choice_based_on_action.ipynb index f4ad3571e..e367d52c2 100644 --- a/docs/examples/select_choice_based_on_action.ipynb +++ b/docs/examples/select_choice_based_on_action.ipynb @@ -730,7 +730,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.17" + "version": "3.11.6" } }, "nbformat": 4, diff --git a/docs/examples/text_summarization_quality.ipynb b/docs/examples/text_summarization_quality.ipynb index 4b09891be..39417f631 100644 --- a/docs/examples/text_summarization_quality.ipynb +++ b/docs/examples/text_summarization_quality.ipynb @@ -344,10 +344,10 @@ "source": [ "import openai\n", "\n", - "raw_llm_response, validated_response = guard(\n", - " openai.Completion.create,\n", + "raw_llm_response, validated_response, *rest = guard(\n", + " openai.completions.create,\n", " prompt_params={'document': document},\n", - " engine='text-davinci-003',\n", + " model='text-davinci-003',\n", " max_tokens=2048,\n", " temperature=0\n", ")\n", @@ -565,7 +565,7 @@ } ], "source": [ - "print(guard.state.most_recent_call.tree)" + "print(guard.history.last.tree)" ] }, { @@ -598,10 +598,10 @@ } ], "source": [ - "raw_llm_response, validated_response = guard(\n", - " openai.Completion.create,\n", + "raw_llm_response, validated_response, *rest = guard(\n", + " openai.completions.create,\n", " prompt_params={'document': open(\"data/article1.txt\", \"r\").read()},\n", - " engine='text-ada-001',\n", + " model='text-ada-001',\n", " max_tokens=512,\n", " temperature=0\n", ")\n", @@ -927,7 +927,7 @@ } ], "source": [ - "print(guard.state.most_recent_call.tree)" + "print(guard.history.last.tree)" ] } ], diff --git a/docs/examples/valid_chess_moves.ipynb b/docs/examples/valid_chess_moves.ipynb index 55d4470a6..73d4f5ec6 100644 --- a/docs/examples/valid_chess_moves.ipynb +++ b/docs/examples/valid_chess_moves.ipynb @@ -559,7 +559,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.3" + "version": "3.11.6" }, "orig_nbformat": 4, "vscode": { diff --git a/docs/examples/value_within_distribution.ipynb b/docs/examples/value_within_distribution.ipynb index 4c5d2f666..9b3e9c89e 100644 --- a/docs/examples/value_within_distribution.ipynb +++ b/docs/examples/value_within_distribution.ipynb @@ -227,7 +227,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.17" + "version": "3.11.6" }, "orig_nbformat": 4 }, diff --git a/guardrails/classes/history/call.py b/guardrails/classes/history/call.py index 90b01e58b..39bc07ab7 100644 --- a/guardrails/classes/history/call.py +++ b/guardrails/classes/history/call.py @@ -18,6 +18,8 @@ gather_reasks, sub_reasks_with_fixed_values, ) +from guardrails.utils.safe_get import get_value_from_path +from guardrails.validator_base import Refrain # We can't inherit from Iteration because python @@ -132,16 +134,6 @@ def reask_instructions(self) -> Stack[str]: return Stack() - # # Since all properties now are cumulative in nature, this has no place here - # @property - # def outputs(self) -> Outputs: - # """The outputs from the last iteration.""" - # last_iteration = self.iterations.last - # if last_iteration: - # return last_iteration.outputs - # # To allow chaining without getting AttributeErrors - # return Outputs() - @property def logs(self) -> Stack[str]: """Returns all logs from all iterations as a stack.""" @@ -313,6 +305,24 @@ def failed_validations(self) -> Stack[ValidatorLogs]: ] ) + def _has_unresolved_failures(self) -> bool: + # Check for unresolved ReAsks + if len(self.reasks) > 0: + return True + + # Check for scenario where no specified on-fail's produced an unfixed ReAsk, + # but valdiation still failed (i.e. Refrain or NoOp). + output = self.fixed_output + for failure in self.failed_validations: + value = get_value_from_path(output, failure.property_path) + if value == failure.value_before_validation or isinstance( + failure.value_after_validation, Refrain + ): + return True + + # No ReAsks and no unresolved failed validations + return False + @property def status(self) -> str: """Returns the cumulative status of the run based on the validity of @@ -321,7 +331,7 @@ def status(self) -> str: return not_run_status elif self.error: return error_status - elif len(self.reasks) > 0: + elif self._has_unresolved_failures(): return fail_status return pass_status diff --git a/guardrails/utils/logs_utils.py b/guardrails/utils/logs_utils.py index eb7a24379..a14eed58b 100644 --- a/guardrails/utils/logs_utils.py +++ b/guardrails/utils/logs_utils.py @@ -13,6 +13,7 @@ class ValidatorLogs(ArbitraryModel): value_before_validation: Any validation_result: Optional[ValidationResult] = None value_after_validation: Optional[Any] = None + property_path: str def update_response_by_path(output: dict, path: List[Any], value: Any) -> None: diff --git a/guardrails/utils/reask_utils.py b/guardrails/utils/reask_utils.py index 678a264e0..4e45f617b 100644 --- a/guardrails/utils/reask_utils.py +++ b/guardrails/utils/reask_utils.py @@ -221,7 +221,10 @@ def sub_reasks_with_fixed_values(value: Any) -> Any: for dict_key, dict_value in value.items(): copy[dict_key] = sub_reasks_with_fixed_values(dict_value) elif isinstance(copy, FieldReAsk): + fix_value = copy.fail_results[0].fix_value # TODO handle multiple fail results - copy = copy.fail_results[0].fix_value + # Leave the ReAsk in place if there is no fix value + # This allows us to determine the proper status for the call + copy = fix_value if fix_value is not None else copy return copy diff --git a/guardrails/utils/safe_get.py b/guardrails/utils/safe_get.py index 02e7ce19f..c5a138958 100644 --- a/guardrails/utils/safe_get.py +++ b/guardrails/utils/safe_get.py @@ -2,7 +2,7 @@ def safe_get_with_brackets( - container: Union[List[Any], Any], key: Any, default: Optional[Any] = None + container: Union[str, List[Any], Any], key: Any, default: Optional[Any] = None ) -> Any: try: value = container[key] @@ -14,9 +14,32 @@ def safe_get_with_brackets( def safe_get( - container: Union[List[Any], Dict[Any, Any]], key: Any, default: Optional[Any] = None + container: Union[str, List[Any], Dict[Any, Any]], + key: Any, + default: Optional[Any] = None, ) -> Any: if isinstance(container, dict): return container.get(key, default) else: return safe_get_with_brackets(container, key, default) + + +def get_value_from_path( + object: Optional[Union[str, List[Any], Dict[Any, Any]]], property_path: str +) -> Any: + if object is None: + return None + + path_elems = property_path.split(".") + path_elems.pop(0) + + value = object + for elem in path_elems: + obj_value = safe_get(value, elem) + if not obj_value and elem.isnumeric(): + # value was empty but the key may be an array index + value = safe_get(value, int(elem)) + else: + value = obj_value + + return value diff --git a/guardrails/validator_service.py b/guardrails/validator_service.py index 4777ee274..293755d8e 100644 --- a/guardrails/validator_service.py +++ b/guardrails/validator_service.py @@ -70,12 +70,18 @@ def perform_correction( ) def run_validator( - self, iteration: Iteration, validator: Validator, value: Any, metadata: Dict + self, + iteration: Iteration, + validator: Validator, + value: Any, + metadata: Dict, + property_path: str, ) -> ValidatorLogs: validator_class_name = validator.__class__.__name__ validator_logs = ValidatorLogs( validator_name=validator_class_name, value_before_validation=value, + property_path=property_path, ) iteration.outputs.validator_logs.append(validator_logs) @@ -94,10 +100,13 @@ def run_validators( validator_setup: FieldValidation, value: Any, metadata: Dict[str, Any], + property_path: str, ) -> Tuple[Any, Dict[str, Any]]: # Validate the field for validator in validator_setup.validators: - validator_logs = self.run_validator(iteration, validator, value, metadata) + validator_logs = self.run_validator( + iteration, validator, value, metadata, property_path + ) result = validator_logs.validation_result if isinstance(result, FailResult): @@ -127,14 +136,12 @@ def validate_dependents( metadata: Dict, validator_setup: FieldValidation, iteration: Iteration, + parent_path: str, ): for child_setup in validator_setup.children: child_schema = safe_get(value, child_setup.key) child_schema, metadata = self.validate( - child_schema, - metadata, - child_setup, - iteration, + child_schema, metadata, child_setup, iteration, parent_path ) value[child_setup.key] = child_schema @@ -144,14 +151,18 @@ def validate( metadata: dict, validator_setup: FieldValidation, iteration: Iteration, + path: str = "$", ) -> Tuple[Any, dict]: + property_path = f"{path}.{validator_setup.key}" # Validate children first if validator_setup.children: - self.validate_dependents(value, metadata, validator_setup, iteration) + self.validate_dependents( + value, metadata, validator_setup, iteration, property_path + ) # Validate the field value, metadata = self.run_validators( - iteration, validator_setup, value, metadata + iteration, validator_setup, value, metadata, property_path ) return value, metadata @@ -186,6 +197,7 @@ async def run_validators( validator_setup: FieldValidation, value: Any, metadata: Dict, + property_path: str, ): loop = asyncio.get_running_loop() for on_fail, validator_group in self.group_validators( @@ -204,11 +216,14 @@ async def run_validators( validator, value, metadata, + property_path, ) ) else: # run the validators in the current process - result = self.run_validator(iteration, validator, value, metadata) + result = self.run_validator( + iteration, validator, value, metadata, property_path + ) validators_logs.append(result) # wait for the parallel tasks to finish @@ -254,14 +269,12 @@ async def validate_dependents( metadata: Dict, validator_setup: FieldValidation, iteration: Iteration, + parent_path: str, ): async def process_child(child_setup): child_value = safe_get(value, child_setup.key) new_child_value, new_metadata = await self.async_validate( - child_value, - metadata, - child_setup, - iteration, + child_value, metadata, child_setup, iteration, parent_path ) return child_setup.key, new_child_value, new_metadata @@ -281,14 +294,18 @@ async def async_validate( metadata: dict, validator_setup: FieldValidation, iteration: Iteration, + path: str = "$", ) -> Tuple[Any, dict]: + property_path = f"{path}.{validator_setup.key}" if validator_setup.key else path # Validate children first if validator_setup.children: - await self.validate_dependents(value, metadata, validator_setup, iteration) + await self.validate_dependents( + value, metadata, validator_setup, iteration, property_path + ) # Validate the field value, metadata = await self.run_validators( - iteration, validator_setup, value, metadata + iteration, validator_setup, value, metadata, property_path ) return value, metadata diff --git a/tests/integration_tests/test_assets/entity_extraction/validated_output_refrain.py b/tests/integration_tests/test_assets/entity_extraction/validated_output_refrain.py index ac92c6d5b..2e941339c 100644 --- a/tests/integration_tests/test_assets/entity_extraction/validated_output_refrain.py +++ b/tests/integration_tests/test_assets/entity_extraction/validated_output_refrain.py @@ -1 +1 @@ -VALIDATED_OUTPUT_REFRAIN = {} +VALIDATED_OUTPUT_REFRAIN = None diff --git a/tests/integration_tests/test_assets/pydantic/validated_response_reask.py b/tests/integration_tests/test_assets/pydantic/validated_response_reask.py index f4a6125da..f7756814d 100644 --- a/tests/integration_tests/test_assets/pydantic/validated_response_reask.py +++ b/tests/integration_tests/test_assets/pydantic/validated_response_reask.py @@ -147,7 +147,24 @@ class ListOfPeople(BaseModel): VALIDATED_OUTPUT_3 = { "people": [ - {"name": "John Doe", "age": 28, "zip_code": None}, + { + "name": "John Doe", + "age": 28, + "zip_code": FieldReAsk( + incorrect_value="None", + fail_results=[ + FailResult( + error_message="Zip code must be numeric.", + fix_value=None, + ), + FailResult( + error_message="Zip code must be in California, and start with 9.", + fix_value=None, + ), + ], + path=["people", 0, "zip_code"], + ), + }, {"name": "Jane Doe", "age": 32, "zip_code": "94103"}, {"name": "James Smith", "age": 40, "zip_code": "92101"}, ] diff --git a/tests/integration_tests/test_async.py b/tests/integration_tests/test_async.py index 623aa7cb6..f35fe39d3 100644 --- a/tests/integration_tests/test_async.py +++ b/tests/integration_tests/test_async.py @@ -45,9 +45,9 @@ async def test_entity_extraction_with_reask(mocker, multiprocessing_validators: mock_preprocess_prompt.assert_called() # Assertions are made on the guard state object. + assert final_output.validation_passed is True assert final_output.validated_output == entity_extraction.VALIDATED_OUTPUT_REASK_2 - # FIXME guard_history = guard.history call = guard_history.first @@ -91,7 +91,13 @@ async def test_entity_extraction_with_noop(mocker): ) # Assertions are made on the guard state object. - assert final_output.validated_output == entity_extraction.VALIDATED_OUTPUT_NOOP + + # Old assertion which is wrong + # This should not pass validation and therefore will not have a validated output + # assert final_output.validated_output == entity_extraction.VALIDATED_OUTPUT_NOOP + + assert final_output.validation_passed is False + assert final_output.validated_output is None call = guard.history.first @@ -103,7 +109,7 @@ async def test_entity_extraction_with_noop(mocker): # For orginal prompt and output assert call.compiled_prompt == entity_extraction.COMPILED_PROMPT assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT - assert call.validated_output == entity_extraction.VALIDATED_OUTPUT_NOOP + assert call.validation_output == entity_extraction.VALIDATED_OUTPUT_NOOP @pytest.mark.asyncio @@ -124,7 +130,8 @@ async def test_entity_extraction_with_noop_pydantic(mocker): ) # Assertions are made on the guard state object. - assert final_output.validated_output == entity_extraction.VALIDATED_OUTPUT_NOOP + assert final_output.validation_passed is False + assert final_output.validated_output is None call = guard.history.first @@ -136,7 +143,7 @@ async def test_entity_extraction_with_noop_pydantic(mocker): # For orginal prompt and output assert call.compiled_prompt == entity_extraction.COMPILED_PROMPT assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT - assert call.validated_output == entity_extraction.VALIDATED_OUTPUT_NOOP + assert call.validation_output == entity_extraction.VALIDATED_OUTPUT_NOOP @pytest.mark.asyncio @@ -157,6 +164,7 @@ async def test_entity_extraction_with_filter(mocker): ) # Assertions are made on the guard state object. + assert final_output.validation_passed is True assert final_output.validated_output == entity_extraction.VALIDATED_OUTPUT_FILTER call = guard.history.first @@ -168,6 +176,7 @@ async def test_entity_extraction_with_filter(mocker): # For orginal prompt and output assert call.compiled_prompt == entity_extraction.COMPILED_PROMPT assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT + assert call.validation_output == entity_extraction.VALIDATED_OUTPUT_FILTER assert call.validated_output == entity_extraction.VALIDATED_OUTPUT_FILTER @@ -189,6 +198,7 @@ async def test_entity_extraction_with_fix(mocker): ) # Assertions are made on the guard state object. + assert final_output.validation_passed is True assert final_output.validated_output == entity_extraction.VALIDATED_OUTPUT_FIX call = guard.history.first @@ -219,6 +229,8 @@ async def test_entity_extraction_with_refrain(mocker): num_reasks=1, ) # Assertions are made on the guard state object. + + assert final_output.validation_passed is False assert final_output.validated_output == entity_extraction.VALIDATED_OUTPUT_REFRAIN call = guard.history.first diff --git a/tests/integration_tests/test_guard.py b/tests/integration_tests/test_guard.py index 9a19b4861..0d736cc80 100644 --- a/tests/integration_tests/test_guard.py +++ b/tests/integration_tests/test_guard.py @@ -223,7 +223,8 @@ def test_entity_extraction_with_noop(mocker, rail, prompt): ) # Assertions are made on the guard state object. - assert final_output.validated_output == entity_extraction.VALIDATED_OUTPUT_NOOP + assert final_output.validation_passed is False + assert final_output.validated_output is None call = guard.history.first @@ -233,7 +234,8 @@ def test_entity_extraction_with_noop(mocker, rail, prompt): # For orginal prompt and output assert call.compiled_prompt == entity_extraction.COMPILED_PROMPT assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT - assert call.validated_output == entity_extraction.VALIDATED_OUTPUT_NOOP + assert call.validated_output is None + assert call.validation_output == entity_extraction.VALIDATED_OUTPUT_NOOP @pytest.mark.parametrize( @@ -745,7 +747,7 @@ def test_in_memory_validator_log_is_not_duplicated(mocker): entity_extraction.PYDANTIC_RAIL_WITH_NOOP, entity_extraction.PYDANTIC_PROMPT ) - _, final_output, *rest = guard( + guard( llm_api=get_static_openai_create_func(), prompt_params={"document": content[:6000]}, num_reasks=1, @@ -756,7 +758,10 @@ def test_in_memory_validator_log_is_not_duplicated(mocker): for x in guard.history.first.iterations.first.validator_logs if x.validator_name == "OneLine" ) - assert len(one_line_logs) == len(final_output.get("fees")) + + assert len(one_line_logs) == len( + guard.history.first.validation_output.get("fees") + ) finally: OneLine.run_in_separate_process = separate_proc_bak diff --git a/tests/integration_tests/test_pydantic.py b/tests/integration_tests/test_pydantic.py index dcdc7cd23..bb7065894 100644 --- a/tests/integration_tests/test_pydantic.py +++ b/tests/integration_tests/test_pydantic.py @@ -31,7 +31,8 @@ def test_pydantic_with_reask(mocker): ) # Assertions are made on the guard state object. - assert final_output.validated_output == pydantic.VALIDATED_OUTPUT_REASK_3 + assert final_output.validation_passed is False + assert final_output.validated_output is None call = guard.history.first @@ -76,7 +77,8 @@ def test_pydantic_with_reask(mocker): # Same as above assert call.reask_prompts.last == pydantic.COMPILED_PROMPT_REASK_2 assert call.raw_outputs.last == pydantic.LLM_OUTPUT_REASK_2 - assert call.validated_output == pydantic.VALIDATED_OUTPUT_REASK_3 + assert call.validated_output is None + assert call.validation_output == pydantic.VALIDATED_OUTPUT_REASK_3 def test_pydantic_with_full_schema_reask(mocker): @@ -86,7 +88,7 @@ def test_pydantic_with_full_schema_reask(mocker): ) guard = gd.Guard.from_pydantic(ListOfPeople, prompt=VALIDATED_RESPONSE_REASK_PROMPT) - _, final_output, *rest = guard( + final_output = guard( get_static_openai_chat_create_func(), model="gpt-3.5-turbo", max_tokens=512, @@ -96,7 +98,8 @@ def test_pydantic_with_full_schema_reask(mocker): ) # Assertions are made on the guard state object. - assert final_output == pydantic.VALIDATED_OUTPUT_REASK_3 + assert final_output.validation_passed is False + assert final_output.validated_output is None call = guard.history.first @@ -127,7 +130,8 @@ def test_pydantic_with_full_schema_reask(mocker): pydantic.COMPILED_INSTRUCTIONS_CHAT ) assert call.raw_outputs.last == pydantic.LLM_OUTPUT_FULL_REASK_2 - assert call.validated_output == pydantic.VALIDATED_OUTPUT_REASK_3 + assert call.validated_output is None + assert call.validation_output == pydantic.VALIDATED_OUTPUT_REASK_3 class ContainerModel(BaseModel): diff --git a/tests/unit_tests/classes/history/test_call.py b/tests/unit_tests/classes/history/test_call.py index fddf9099c..2210f9e72 100644 --- a/tests/unit_tests/classes/history/test_call.py +++ b/tests/unit_tests/classes/history/test_call.py @@ -105,6 +105,7 @@ def custom_llm(): value_before_validation="Hello there!", validation_result=first_validation_result, value_after_validation="Hello there", + property_path="$", ) first_validator_logs = [first_validator_log] first_outputs = Outputs( @@ -140,6 +141,7 @@ def custom_llm(): value_before_validation="Hello there", validation_result=PassResult(), value_after_validation="Hello there", + property_path="$", ) second_validator_logs = [second_validator_log] second_outputs = Outputs( diff --git a/tests/unit_tests/classes/history/test_iteration.py b/tests/unit_tests/classes/history/test_iteration.py index e1aa09d24..d8d0fdee4 100644 --- a/tests/unit_tests/classes/history/test_iteration.py +++ b/tests/unit_tests/classes/history/test_iteration.py @@ -80,6 +80,7 @@ def test_non_empty_initialization(): value_before_validation="Hello there!", validation_result=validation_result, value_after_validation="Hello there", + property_path="$", ) ] error = "Validation Failed!" diff --git a/tests/unit_tests/classes/history/test_outputs.py b/tests/unit_tests/classes/history/test_outputs.py index c6a9ab569..4a43e1a38 100644 --- a/tests/unit_tests/classes/history/test_outputs.py +++ b/tests/unit_tests/classes/history/test_outputs.py @@ -40,6 +40,7 @@ def test_non_empty_initialization(): value_before_validation="Hello there!", validation_result=validation_result, value_after_validation="Hello there", + property_path="$", ) ] error = "Validation Failed!" @@ -106,6 +107,7 @@ def test_non_empty_initialization(): value_before_validation="Hello there!", validation_result=fixable_fail_result, value_after_validation="Hello there", + property_path="$", ) ] ), @@ -127,12 +129,14 @@ def test_failed_validations(): value_before_validation="Hello there!", validation_result=fixable_fail_result, value_after_validation="Hello there", + property_path="$", ), ValidatorLogs( validator_name="valid-length", value_before_validation="Hello there!", validation_result=PassResult(), value_after_validation="Hello there!", + property_path="$", ), ] outputs = Outputs(validator_logs=validator_logs) @@ -153,6 +157,7 @@ def test_failed_validations(): value_before_validation="Hello there!", validation_result=non_fixable_fail_result, value_after_validation="Hello there", + property_path="$", ) ], reasks=[ diff --git a/tests/unit_tests/test_async_validator_service.py b/tests/unit_tests/test_async_validator_service.py index 0f05f675b..8b52db16f 100644 --- a/tests/unit_tests/test_async_validator_service.py +++ b/tests/unit_tests/test_async_validator_service.py @@ -85,11 +85,13 @@ async def test_async_validate_with_children(mocker): assert validate_dependents_mock.call_count == 1 validate_dependents_mock.assert_called_once_with( - True, {}, field_validation, iteration + True, {}, field_validation, iteration, "$.mock-parent-key" ) assert run_validators_mock.call_count == 1 - run_validators_mock.assert_called_once_with(iteration, field_validation, True, {}) + run_validators_mock.assert_called_once_with( + iteration, field_validation, True, {}, "$.mock-parent-key" + ) assert validated_value == "run_validators_mock" assert validated_metadata == {"async": True} @@ -115,7 +117,7 @@ async def test_async_validate_without_children(mocker): assert run_validators_mock.call_count == 1 run_validators_mock.assert_called_once_with( - iteration, empty_field_validation, True, {} + iteration, empty_field_validation, True, {}, "$.mock-key" ) assert validated_value == "run_validators_mock" @@ -152,13 +154,14 @@ async def mock_async_validate(v, md, *args): metadata={}, validator_setup=field_validation, iteration=iteration, + parent_path="$", ) assert gather_spy.call_count == 1 assert async_validate_mock.call_count == 2 - async_validate_mock.assert_any_call(child_one.value, {}, child_one, iteration) - async_validate_mock.assert_any_call(child_two.value, {}, child_two, iteration) + async_validate_mock.assert_any_call(child_one.value, {}, child_one, iteration, "$") + async_validate_mock.assert_any_call(child_two.value, {}, child_two, iteration, "$") assert validated_value == { "child-one-key": "new-child-one-value", @@ -182,11 +185,12 @@ async def test_run_validators(mocker): ("noop", [noop_validator_1, noop_validator_2]), ] - def mock_run_validator(iteration, validator, value, metadata): + def mock_run_validator(iteration, validator, value, metadata, property_path): return ValidatorLogs( validator_name=validator.name, value_before_validation=value, validation_result=PassResult(), + property_path=property_path, ) run_validator_mock = mocker.patch.object( @@ -211,6 +215,7 @@ async def mock_gather(*args): metadata={}, validator_setup=empty_field_validation, iteration=iteration, + property_path="$", ) assert get_running_loop_mock.call_count == 1 @@ -226,6 +231,7 @@ async def mock_gather(*args): noop_validator_2, empty_field_validation.value, {}, + "$", ) assert run_validator_mock.call_count == 3 @@ -250,6 +256,7 @@ async def test_run_validators_with_override(mocker): validator_name="override", value_before_validation="mock-value", validation_result=PassResult(value_override="override"), + property_path="$", ) mock_loop = MockLoop(True) @@ -267,6 +274,7 @@ async def test_run_validators_with_override(mocker): metadata={}, validator_setup=empty_field_validation, iteration=iteration, + property_path="$", ) assert get_running_loop_mock.call_count == 1 diff --git a/tests/unit_tests/test_validators.py b/tests/unit_tests/test_validators.py index abcc126b2..ea2c55117 100644 --- a/tests/unit_tests/test_validators.py +++ b/tests/unit_tests/test_validators.py @@ -645,7 +645,7 @@ def custom_refrain_on_fail_handler(value: Any, fail_results: List[FailResult]): ), ( custom_refrain_on_fail_handler, - {}, + None, ), ], )