From 5352f0113a9945775562fac1dc1c6fcd48cf251d Mon Sep 17 00:00:00 2001 From: Josh Reini Date: Fri, 25 Oct 2024 12:48:39 -0400 Subject: [PATCH] TruLens v1.1.0 (#1539) * drafting * undo file add * version bump * poetry lock * apply groundedness reasons normalization to answerability versino * fix AttributeError * fix engine URL * blog updates * missing emojis * poetry lock --------- Co-authored-by: Josh Reini Co-authored-by: corey hu --- .../posts/trulens_1_1_dashboard_updates.md | 22 ++++---- poetry.lock | 56 ++++++++++--------- pyproject.toml | 2 +- src/apps/langchain/pyproject.toml | 2 +- src/apps/llamaindex/pyproject.toml | 2 +- src/apps/nemo/pyproject.toml | 2 +- src/benchmark/pyproject.toml | 2 +- src/connectors/snowflake/pyproject.toml | 2 +- .../trulens/connectors/snowflake/connector.py | 2 +- src/core/pyproject.toml | 2 +- src/dashboard/pyproject.toml | 2 +- src/feedback/pyproject.toml | 2 +- src/feedback/trulens/feedback/llm_provider.py | 4 +- src/providers/bedrock/pyproject.toml | 2 +- src/providers/cortex/pyproject.toml | 2 +- src/providers/huggingface/pyproject.toml | 2 +- src/providers/langchain/pyproject.toml | 2 +- src/providers/litellm/pyproject.toml | 2 +- src/providers/openai/pyproject.toml | 2 +- src/trulens_eval/pyproject.toml | 2 +- 20 files changed, 61 insertions(+), 55 deletions(-) diff --git a/docs/blog/posts/trulens_1_1_dashboard_updates.md b/docs/blog/posts/trulens_1_1_dashboard_updates.md index 713e7800c..2c7451083 100644 --- a/docs/blog/posts/trulens_1_1_dashboard_updates.md +++ b/docs/blog/posts/trulens_1_1_dashboard_updates.md @@ -1,13 +1,12 @@ --- categories: - General -date: 2024-09-25 +date: 2024-10-09 --- # What's new in TruLens 1.1: Dashboard Comparison View, Multi-App Support, Metadata Editing, and More! -TruLens 1.1.0 has been released! This release includes a number of improvements to the TruLens dashboard, including a new comparison view and a more intuitive user interface. We have also made several improvements performance and usability. - +In TruLens 1.1, we re-imagined the dashboard with a focus on making it easy to track large numbers of experiments, make comparisons and improve your apps for production. We also made several improvements performance and usability. ## Dashboard Highlights @@ -20,7 +19,7 @@ An overhaul of the TruLens dashboard has been released with major features and i #### Global app selector -TruLens 1.0 introduced app versioning, allowing performance of their LLM apps to be tracked across different versions. On multi-app tables, the dashboard sidebar now includes an app selector to quickly navigate to the desired application. +TruLens 1.0 introduced app versioning, allowing performance of their LLM apps to be tracked across different versions. Now in 1.1 when you're tracking more than one app, the dashboard sidebar now includes an app selector to quickly navigate to the desired application. #### App version and Record search and filtering @@ -84,13 +83,14 @@ The records page has been updated to include a more intuitive flow for viewing a #### Try it out! -We hope you enjoy the new features and improvements in TruLens 1.1.0! To get started, use [`run_dashboard`][trulens.dashboard.run.run_dashboard] with a TruSession object: +We hope you enjoy the new features and improvements in TruLens 1.1! To get started, use [`run_dashboard`][trulens.dashboard.run.run_dashboard] with a TruSession object: +!!! example -```python -from trulens.core import TruSession -from trulens.dashboard import run_dashboard + ```python + from trulens.core import TruSession + from trulens.dashboard import run_dashboard -session = TruSession(...) -run_dashboard(session) -``` + session = TruSession(...) + run_dashboard(session) + ``` diff --git a/poetry.lock b/poetry.lock index db7cac06c..dc0907882 100644 --- a/poetry.lock +++ b/poetry.lock @@ -525,12 +525,12 @@ description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.35.39-py3-none-any.whl", hash = "sha256:5970b62c1ec8177501e02520f0d41839ca5fc549b30bac4e8c0c0882ae776217"}, - {file = "boto3-1.35.39.tar.gz", hash = "sha256:670f811c65e3c5fe4ed8c8d69be0b44b1d649e992c0fc16de43816d1188f88f1"}, + {file = "boto3-1.35.37-py3-none-any.whl", hash = "sha256:385ca77bf8ea4ab2d97f6e2435bdb29f77d9301e2f7ac796c2f465753c2adf3c"}, + {file = "boto3-1.35.37.tar.gz", hash = "sha256:470d981583885859fed2fd1c185eeb01cc03e60272d499bafe41b12625b158c8"}, ] [package.dependencies] -botocore = ">=1.35.39,<1.36.0" +botocore = ">=1.35.37,<1.36.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -4310,12 +4310,12 @@ test = ["pep440", "pre-commit", "pytest", "testpath"] [[package]] name = "nemoguardrails" -version = "0.9.1.1" -description = "NeMo Guardrails is an open-source toolkit for easily adding programmagle guardrails to LLM-based conversational systems." +version = "0.10.1" +description = "NeMo Guardrails is an open-source toolkit for easily adding programmable guardrails to LLM-based conversational systems." optional = false python-versions = ">=3.8" files = [ - {file = "nemoguardrails-0.9.1.1-py3-none-any.whl", hash = "sha256:2d6aed9c7ecec9434b3018ffbbfc863b1c51141bd2086443361b9ea73cd1ac03"}, + {file = "nemoguardrails-0.10.1-py3-none-any.whl", hash = "sha256:11b7db76337365adea33567d6a35299b34903174253083ac984d1a16953e2fc2"}, ] [package.dependencies] @@ -4324,10 +4324,10 @@ annoy = ">=1.17.3" fastapi = ">=0.103.0" fastembed = ">=0.2.2" httpx = ">=0.24.1" -jinja2 = ">=3.1.3" -langchain = ">=0.1.0,<0.1.9 || >0.1.9,<0.3.0" +jinja2 = ">=3.1.4" +langchain = ">=0.2.14,<0.3.0" langchain-community = ">=0.0.16,<0.3.0" -langchain-core = ">=0.1.0,<0.1.26 || >0.1.26,<0.3.0" +langchain-core = ">=0.2.14,<0.3.0" lark = ">=1.1.7,<1.2.0" nest-asyncio = ">=1.5.6" prompt-toolkit = ">=3.0" @@ -4341,9 +4341,10 @@ uvicorn = ">=0.23" watchdog = ">=3.0.0" [package.extras] -all = ["nemoguardrails[eval,openai,sdd]"] -dev = ["aioresponses (>=0.7.6)", "black (==23.3.0)", "mypy (>=1.1.1)", "pre-commit (>=3.1.1)", "pylint (>=2.17.0)", "pytest (>=7.2.2)", "pytest-asyncio (>=0.21.0)", "pytest-cov (>=4.1.0)", "pytest-httpx (>=0.22.0)"] -eval = ["numpy (>=1.24,<2.0)", "tqdm (>=4.65,<5.0)"] +all = ["nemoguardrails[eval,gcp,openai,sdd]"] +dev = ["aioresponses (>=0.7.6)", "black (==23.3.0)", "mypy (>=1.1.1)", "pre-commit (>=3.1.1)", "pylint (>=2.17.0)", "pytest (>=7.2.2)", "pytest-asyncio (>=0.21.0)", "pytest-cov (>=4.1.0)", "pytest-httpx (>=0.22.0,<0.32.0)", "streamlit (>=1.37.0)"] +eval = ["numpy (>=1.24,<2.0)", "streamlit (>=1.37.0)", "tqdm (>=4.65,<5.0)"] +gcp = ["google-cloud-language (>=2.14.0)"] openai = ["langchain-openai (>=0.0.5)"] sdd = ["presidio-analyzer (>=2.2)", "presidio-anonymizer (>=2.2)", "spacy (>=3.7.2)"] @@ -6670,6 +6671,11 @@ files = [ {file = "scikit_learn-1.5.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f60021ec1574e56632be2a36b946f8143bf4e5e6af4a06d85281adc22938e0dd"}, {file = "scikit_learn-1.5.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:394397841449853c2290a32050382edaec3da89e35b3e03d6cc966aebc6a8ae6"}, {file = "scikit_learn-1.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:57cc1786cfd6bd118220a92ede80270132aa353647684efa385a74244a41e3b1"}, + {file = "scikit_learn-1.5.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:e9a702e2de732bbb20d3bad29ebd77fc05a6b427dc49964300340e4c9328b3f5"}, + {file = "scikit_learn-1.5.2-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:b0768ad641981f5d3a198430a1d31c3e044ed2e8a6f22166b4d546a5116d7908"}, + {file = "scikit_learn-1.5.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:178ddd0a5cb0044464fc1bfc4cca5b1833bfc7bb022d70b05db8530da4bb3dd3"}, + {file = "scikit_learn-1.5.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f7284ade780084d94505632241bf78c44ab3b6f1e8ccab3d2af58e0e950f9c12"}, + {file = "scikit_learn-1.5.2-cp313-cp313-win_amd64.whl", hash = "sha256:b7b0f9a0b1040830d38c39b91b3a44e1b643f4b36e36567b80b7c6bd2202a27f"}, {file = "scikit_learn-1.5.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:757c7d514ddb00ae249832fe87100d9c73c6ea91423802872d9e74970a0e40b9"}, {file = "scikit_learn-1.5.2-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:52788f48b5d8bca5c0736c175fa6bdaab2ef00a8f536cda698db61bd89c551c1"}, {file = "scikit_learn-1.5.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:643964678f4b5fbdc95cbf8aec638acc7aa70f5f79ee2cdad1eec3df4ba6ead8"}, @@ -7873,7 +7879,7 @@ tutorials = ["matplotlib", "pandas", "tabulate"] [[package]] name = "trulens-apps-langchain" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." optional = false python-versions = "^3.9,<3.13" @@ -7892,7 +7898,7 @@ url = "src/apps/langchain" [[package]] name = "trulens-apps-llamaindex" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." optional = false python-versions = "^3.9" @@ -7911,7 +7917,7 @@ url = "src/apps/llamaindex" [[package]] name = "trulens-apps-nemo" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." optional = false python-versions = ">=3.9,<3.12" @@ -7930,7 +7936,7 @@ url = "src/apps/nemo" [[package]] name = "trulens-benchmark" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." optional = false python-versions = "^3.9" @@ -7946,7 +7952,7 @@ url = "src/benchmark" [[package]] name = "trulens-connectors-snowflake" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." optional = false python-versions = ">=3.9,<3.12" @@ -7992,7 +7998,7 @@ url = "src/core" [[package]] name = "trulens-dashboard" -version = "1.0.2a0" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." optional = false python-versions = "^3.9,!=3.9.7" @@ -8019,7 +8025,7 @@ url = "src/dashboard" [[package]] name = "trulens-eval" -version = "1.0.1" +version = "1.1.0" description = "Backwards-compatibility package for API of trulens_eval<1.0.0 using API of trulens-*>=1.0.0." optional = false python-versions = "^3.9,!=3.9.7" @@ -8035,7 +8041,7 @@ url = "src/trulens_eval" [[package]] name = "trulens-feedback" -version = "1.0.1" +version = "1.1.0" description = "A TruLens extension package implementing feedback functions for LLM App evaluation." optional = false python-versions = "^3.9" @@ -8056,7 +8062,7 @@ url = "src/feedback" [[package]] name = "trulens-providers-bedrock" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." optional = false python-versions = "^3.9" @@ -8075,7 +8081,7 @@ url = "src/providers/bedrock" [[package]] name = "trulens-providers-cortex" -version = "1.0.1" +version = "1.1.0" description = "A TruLens extension package adding Snowflake Cortex support for LLM App evaluation." optional = false python-versions = ">=3.9,<3.12" @@ -8094,7 +8100,7 @@ url = "src/providers/cortex" [[package]] name = "trulens-providers-huggingface" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." optional = false python-versions = "^3.9" @@ -8116,7 +8122,7 @@ url = "src/providers/huggingface" [[package]] name = "trulens-providers-langchain" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." optional = false python-versions = "^3.9" @@ -8134,7 +8140,7 @@ url = "src/providers/langchain" [[package]] name = "trulens-providers-litellm" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." optional = false python-versions = "^3.9" diff --git a/pyproject.toml b/pyproject.toml index d7dcd8e94..78dbe45d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ requires = [ [tool.poetry] name = "trulens" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." authors = [ "Snowflake Inc. ", diff --git a/src/apps/langchain/pyproject.toml b/src/apps/langchain/pyproject.toml index 3cef2d849..e8a7c5258 100644 --- a/src/apps/langchain/pyproject.toml +++ b/src/apps/langchain/pyproject.toml @@ -6,7 +6,7 @@ requires = [ [tool.poetry] name = "trulens-apps-langchain" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." authors = [ "Snowflake Inc. ", diff --git a/src/apps/llamaindex/pyproject.toml b/src/apps/llamaindex/pyproject.toml index 54d1ead77..e4d55ea22 100644 --- a/src/apps/llamaindex/pyproject.toml +++ b/src/apps/llamaindex/pyproject.toml @@ -6,7 +6,7 @@ requires = [ [tool.poetry] name = "trulens-apps-llamaindex" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." authors = [ "Snowflake Inc. ", diff --git a/src/apps/nemo/pyproject.toml b/src/apps/nemo/pyproject.toml index 936d074f1..27cec85d7 100644 --- a/src/apps/nemo/pyproject.toml +++ b/src/apps/nemo/pyproject.toml @@ -6,7 +6,7 @@ requires = [ [tool.poetry] name = "trulens-apps-nemo" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." authors = [ "Snowflake Inc. ", diff --git a/src/benchmark/pyproject.toml b/src/benchmark/pyproject.toml index 4a291ddb1..831c45b99 100644 --- a/src/benchmark/pyproject.toml +++ b/src/benchmark/pyproject.toml @@ -6,7 +6,7 @@ requires = [ [tool.poetry] name = "trulens-benchmark" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." authors = [ "Snowflake Inc. ", diff --git a/src/connectors/snowflake/pyproject.toml b/src/connectors/snowflake/pyproject.toml index 5902caeab..56e557692 100644 --- a/src/connectors/snowflake/pyproject.toml +++ b/src/connectors/snowflake/pyproject.toml @@ -6,7 +6,7 @@ requires = [ [tool.poetry] name = "trulens-connectors-snowflake" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." authors = [ "Snowflake Inc. ", diff --git a/src/connectors/snowflake/trulens/connectors/snowflake/connector.py b/src/connectors/snowflake/trulens/connectors/snowflake/connector.py index 18d8edd64..f0bce9a45 100644 --- a/src/connectors/snowflake/trulens/connectors/snowflake/connector.py +++ b/src/connectors/snowflake/trulens/connectors/snowflake/connector.py @@ -74,7 +74,7 @@ def __init__( database_prefix, database_args, database_check_revision, - URL(**connection_parameters), + connection_parameters=connection_parameters, ) def _create_snowpark_session( diff --git a/src/core/pyproject.toml b/src/core/pyproject.toml index 11cd4ede4..67ead86e0 100644 --- a/src/core/pyproject.toml +++ b/src/core/pyproject.toml @@ -6,7 +6,7 @@ requires = [ [tool.poetry] name = "trulens-core" -version = "1.0.2a0" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." authors = [ "Snowflake Inc. ", diff --git a/src/dashboard/pyproject.toml b/src/dashboard/pyproject.toml index 6ae363c80..68ee14bff 100644 --- a/src/dashboard/pyproject.toml +++ b/src/dashboard/pyproject.toml @@ -6,7 +6,7 @@ requires = [ [tool.poetry] name = "trulens-dashboard" -version = "1.0.2a0" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." authors = [ "Snowflake Inc. ", diff --git a/src/feedback/pyproject.toml b/src/feedback/pyproject.toml index 66470c38c..82f4fd8b8 100644 --- a/src/feedback/pyproject.toml +++ b/src/feedback/pyproject.toml @@ -6,7 +6,7 @@ requires = [ [tool.poetry] name = "trulens-feedback" -version = "1.0.1" +version = "1.1.0" description = "A TruLens extension package implementing feedback functions for LLM App evaluation." authors = [ "Snowflake Inc. ", diff --git a/src/feedback/trulens/feedback/llm_provider.py b/src/feedback/trulens/feedback/llm_provider.py index 48c43a972..bb5e0d571 100644 --- a/src/feedback/trulens/feedback/llm_provider.py +++ b/src/feedback/trulens/feedback/llm_provider.py @@ -1843,8 +1843,8 @@ def evaluate_hypothesis(index, hypothesis): premise=f"{source}", hypothesis=f"{hypothesis}" ) score, reason = self.generate_score_and_reasons( - system_prompt, - user_prompt, + system_prompt=system_prompt, + user_prompt=user_prompt, min_score_val=min_score_val, max_score_val=max_score_val, temperature=temperature, diff --git a/src/providers/bedrock/pyproject.toml b/src/providers/bedrock/pyproject.toml index c0b9e7fda..e3dcad406 100644 --- a/src/providers/bedrock/pyproject.toml +++ b/src/providers/bedrock/pyproject.toml @@ -6,7 +6,7 @@ requires = [ [tool.poetry] name = "trulens-providers-bedrock" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." authors = [ "Snowflake Inc. ", diff --git a/src/providers/cortex/pyproject.toml b/src/providers/cortex/pyproject.toml index 745b034d6..26d267c8f 100644 --- a/src/providers/cortex/pyproject.toml +++ b/src/providers/cortex/pyproject.toml @@ -6,7 +6,7 @@ requires = [ [tool.poetry] name = "trulens-providers-cortex" -version = "1.0.1" +version = "1.1.0" description = "A TruLens extension package adding Snowflake Cortex support for LLM App evaluation." authors = [ "Snowflake Inc. ", diff --git a/src/providers/huggingface/pyproject.toml b/src/providers/huggingface/pyproject.toml index 5740c7f11..9cac27896 100644 --- a/src/providers/huggingface/pyproject.toml +++ b/src/providers/huggingface/pyproject.toml @@ -6,7 +6,7 @@ requires = [ [tool.poetry] name = "trulens-providers-huggingface" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." authors = [ "Snowflake Inc. ", diff --git a/src/providers/langchain/pyproject.toml b/src/providers/langchain/pyproject.toml index 7e456eecc..5749893a3 100644 --- a/src/providers/langchain/pyproject.toml +++ b/src/providers/langchain/pyproject.toml @@ -6,7 +6,7 @@ requires = [ [tool.poetry] name = "trulens-providers-langchain" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." authors = [ "Snowflake Inc. ", diff --git a/src/providers/litellm/pyproject.toml b/src/providers/litellm/pyproject.toml index 6726c8b0e..3da522912 100644 --- a/src/providers/litellm/pyproject.toml +++ b/src/providers/litellm/pyproject.toml @@ -6,7 +6,7 @@ requires = [ [tool.poetry] name = "trulens-providers-litellm" -version = "1.0.1" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." authors = [ "Snowflake Inc. ", diff --git a/src/providers/openai/pyproject.toml b/src/providers/openai/pyproject.toml index 72f47d5ca..b6bb58764 100644 --- a/src/providers/openai/pyproject.toml +++ b/src/providers/openai/pyproject.toml @@ -6,7 +6,7 @@ requires = [ [tool.poetry] name = "trulens-providers-openai" -version = "1.0.2a0" +version = "1.1.0" description = "Library to systematically track and evaluate LLM based applications." authors = [ "Snowflake Inc. ", diff --git a/src/trulens_eval/pyproject.toml b/src/trulens_eval/pyproject.toml index 0701ba8eb..2a09b6b1d 100644 --- a/src/trulens_eval/pyproject.toml +++ b/src/trulens_eval/pyproject.toml @@ -6,7 +6,7 @@ requires = [ [tool.poetry] name = "trulens_eval" -version = "1.0.1" +version = "1.1.0" description = "Backwards-compatibility package for API of trulens_eval<1.0.0 using API of trulens-*>=1.0.0." authors = [ "Snowflake Inc. ",