Merge pull request #526 from guardrails-ai/status-fix

Fix Status

.github/workflows/ci.yml

@@ -173,7 +173,7 @@ jobs:
         - name: Install dependencies
           run: |
             make full;
-            poetry add openai==0.28.1 jupyter nbconvert cohere;
+            poetry add "openai>=1.2.4" jupyter nbconvert cohere;
         - name: Check for pypdfium2
           run: poetry run pip show pypdfium2
         - name: Huggingface Hub Login

.github/workflows/examples_check.yml

@@ -38,7 +38,7 @@ jobs:
     - name: Install dependencies
       run: |
         make full;
-        poetry add openai==0.28.1 jupyter nbconvert cohere;
+        poetry add "openai>=1.2.4" jupyter nbconvert cohere;
     - name: Check for pypdfium2
       run: poetry run pip show pypdfium2
     - name: Huggingface Hub Login
@@ -48,7 +48,7 @@ jobs:
         mkdir /tmp/nltk_data;
         poetry run python -m nltk.downloader -d /tmp/nltk_data punkt;
     - name: Use venv
-      run: source .venv/bin/activate
+      run: source $VENV
     - name: Execute notebooks and check for errors
       run: bash ./.github/workflows/scripts/run_notebooks.sh ${{ matrix.notebook }}
 

docs/examples/extracting_entities.ipynb

@@ -342,7 +342,7 @@
    "source": [
     "import openai\n",
     "\n",
-    "validated_response = guard(\n",
+    "_, validated_response, *rest = guard(\n",
     "    openai.chat.completions.create,\n",
     "    prompt_params={\"document\": content[:6000]},\n",
     "    max_tokens=2048,\n",
@@ -1724,7 +1724,7 @@
     }
    ],
    "source": [
-    "guard.state.most_recent_call.tree"
+    "guard.history.last.tree"
    ]
   }
  ],

docs/examples/generate_structured_data.ipynb

@@ -279,7 +279,7 @@
     "import openai\n",
     "\n",
     "\n",
-    "validated_output = guard(\n",
+    "_, validated_output, *rest = guard(\n",
     "    openai.chat.completions.create,\n",
     "    max_tokens=2048,\n",
     "    temperature=0\n",

docs/examples/guardrails_with_chat_models.ipynb

@@ -1712,7 +1712,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.11.6"
   },
   "orig_nbformat": 4
  },

docs/examples/input_validation.ipynb

@@ -136,7 +136,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.11.6"
   }
  },
  "nbformat": 4,

docs/examples/no_secrets_in_generated_text.ipynb

@@ -293,6 +293,19 @@
     "We can see that the output is a dictionary with the correct schema and types."
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import openai\n",
+    "\n",
+    "raw_llm_response, validated_response, *rest = guard(\n",
+    "   openai.completions.create, model=\"text-davinci-003\", max_tokens=2048, temperature=0\n",
+    ")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 9,

docs/examples/response_is_on_topic.ipynb

@@ -279,7 +279,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.11.6"
   }
  },
  "nbformat": 4,

docs/examples/select_choice_based_on_action.ipynb

@@ -730,7 +730,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.11.6"
   }
  },
  "nbformat": 4,

docs/examples/text_summarization_quality.ipynb

@@ -344,10 +344,10 @@
    "source": [
     "import openai\n",
     "\n",
-    "raw_llm_response, validated_response = guard(\n",
-    "    openai.Completion.create,\n",
+    "raw_llm_response, validated_response, *rest = guard(\n",
+    "    openai.completions.create,\n",
     "    prompt_params={'document': document},\n",
-    "    engine='text-davinci-003',\n",
+    "    model='text-davinci-003',\n",
     "    max_tokens=2048,\n",
     "    temperature=0\n",
     ")\n",
@@ -565,7 +565,7 @@
     }
    ],
    "source": [
-    "print(guard.state.most_recent_call.tree)"
+    "print(guard.history.last.tree)"
    ]
   },
   {
@@ -598,10 +598,10 @@
     }
    ],
    "source": [
-    "raw_llm_response, validated_response = guard(\n",
-    "    openai.Completion.create,\n",
+    "raw_llm_response, validated_response, *rest = guard(\n",
+    "    openai.completions.create,\n",
     "    prompt_params={'document': open(\"data/article1.txt\", \"r\").read()},\n",
-    "    engine='text-ada-001',\n",
+    "    model='text-ada-001',\n",
     "    max_tokens=512,\n",
     "    temperature=0\n",
     ")\n",
@@ -927,7 +927,7 @@
     }
    ],
    "source": [
-    "print(guard.state.most_recent_call.tree)"
+    "print(guard.history.last.tree)"
    ]
   }
  ],

docs/examples/valid_chess_moves.ipynb

@@ -559,7 +559,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.3"
+   "version": "3.11.6"
   },
   "orig_nbformat": 4,
   "vscode": {

docs/examples/value_within_distribution.ipynb

@@ -227,7 +227,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.17"
+   "version": "3.11.6"
   },
   "orig_nbformat": 4
  },

guardrails/classes/history/call.py

@@ -18,6 +18,8 @@
     gather_reasks,
     sub_reasks_with_fixed_values,
 )
+from guardrails.utils.safe_get import get_value_from_path
+from guardrails.validator_base import Refrain
 
 
 # We can't inherit from Iteration because python
@@ -132,16 +134,6 @@ def reask_instructions(self) -> Stack[str]:
 
         return Stack()
 
-    # # Since all properties now are cumulative in nature, this has no place here
-    # @property
-    # def outputs(self) -> Outputs:
-    #     """The outputs from the last iteration."""
-    #     last_iteration = self.iterations.last
-    #     if last_iteration:
-    #         return last_iteration.outputs
-    #     # To allow chaining without getting AttributeErrors
-    #     return Outputs()
-
     @property
     def logs(self) -> Stack[str]:
         """Returns all logs from all iterations as a stack."""
@@ -313,6 +305,24 @@ def failed_validations(self) -> Stack[ValidatorLogs]:
             ]
         )
 
+    def _has_unresolved_failures(self) -> bool:
+        # Check for unresolved ReAsks
+        if len(self.reasks) > 0:
+            return True
+
+        # Check for scenario where no specified on-fail's produced an unfixed ReAsk,
+        #   but valdiation still failed (i.e. Refrain or NoOp).
+        output = self.fixed_output
+        for failure in self.failed_validations:
+            value = get_value_from_path(output, failure.property_path)
+            if value == failure.value_before_validation or isinstance(
+                failure.value_after_validation, Refrain
+            ):
+                return True
+
+        # No ReAsks and no unresolved failed validations
+        return False
+
     @property
     def status(self) -> str:
         """Returns the cumulative status of the run based on the validity of
@@ -321,7 +331,7 @@ def status(self) -> str:
             return not_run_status
         elif self.error:
             return error_status
-        elif len(self.reasks) > 0:
+        elif self._has_unresolved_failures():
             return fail_status
         return pass_status
 

guardrails/utils/logs_utils.py

@@ -13,6 +13,7 @@ class ValidatorLogs(ArbitraryModel):
     value_before_validation: Any
     validation_result: Optional[ValidationResult] = None
     value_after_validation: Optional[Any] = None
+    property_path: str
 
 
 def update_response_by_path(output: dict, path: List[Any], value: Any) -> None:

guardrails/utils/reask_utils.py

@@ -221,7 +221,10 @@ def sub_reasks_with_fixed_values(value: Any) -> Any:
         for dict_key, dict_value in value.items():
             copy[dict_key] = sub_reasks_with_fixed_values(dict_value)
     elif isinstance(copy, FieldReAsk):
+        fix_value = copy.fail_results[0].fix_value
         # TODO handle multiple fail results
-        copy = copy.fail_results[0].fix_value
+        # Leave the ReAsk in place if there is no fix value
+        # This allows us to determine the proper status for the call
+        copy = fix_value if fix_value is not None else copy
 
     return copy

guardrails/utils/safe_get.py

@@ -2,7 +2,7 @@
 
 
 def safe_get_with_brackets(
-    container: Union[List[Any], Any], key: Any, default: Optional[Any] = None
+    container: Union[str, List[Any], Any], key: Any, default: Optional[Any] = None
 ) -> Any:
     try:
         value = container[key]
@@ -14,9 +14,32 @@ def safe_get_with_brackets(
 
 
 def safe_get(
-    container: Union[List[Any], Dict[Any, Any]], key: Any, default: Optional[Any] = None
+    container: Union[str, List[Any], Dict[Any, Any]],
+    key: Any,
+    default: Optional[Any] = None,
 ) -> Any:
     if isinstance(container, dict):
         return container.get(key, default)
     else:
         return safe_get_with_brackets(container, key, default)
+
+
+def get_value_from_path(
+    object: Optional[Union[str, List[Any], Dict[Any, Any]]], property_path: str
+) -> Any:
+    if object is None:
+        return None
+
+    path_elems = property_path.split(".")
+    path_elems.pop(0)
+
+    value = object
+    for elem in path_elems:
+        obj_value = safe_get(value, elem)
+        if not obj_value and elem.isnumeric():
+            # value was empty but the key may be an array index
+            value = safe_get(value, int(elem))
+        else:
+            value = obj_value
+
+    return value