diff --git a/.azure_pipelines/ci-eval-pr.yaml b/.azure_pipelines/ci-eval-pr.yaml index 5ada26a84..08642e241 100644 --- a/.azure_pipelines/ci-eval-pr.yaml +++ b/.azure_pipelines/ci-eval-pr.yaml @@ -37,6 +37,7 @@ jobs: set -e source activate $(condaEnvFileSuffix) which python + python --version displayName: Which Python - bash: | diff --git a/.azure_pipelines/ci-eval.yaml b/.azure_pipelines/ci-eval.yaml index e35cbc367..32afd79ae 100644 --- a/.azure_pipelines/ci-eval.yaml +++ b/.azure_pipelines/ci-eval.yaml @@ -53,6 +53,7 @@ jobs: set -e source activate $(condaEnvFileSuffix) which python + python --version displayName: Which Python - bash: | diff --git a/.azure_pipelines/ci.yaml b/.azure_pipelines/ci.yaml index 1e7af009a..15515f904 100644 --- a/.azure_pipelines/ci.yaml +++ b/.azure_pipelines/ci.yaml @@ -79,6 +79,7 @@ jobs: set -e source activate $(condaEnvFileSuffix) which python + python --version displayName: Which Python - bash: | set -e diff --git a/docs/trulens_eval/langchain_instrumentation.ipynb b/docs/trulens_eval/langchain_instrumentation.ipynb index 83cf21035..6b1cacdee 100644 --- a/docs/trulens_eval/langchain_instrumentation.ipynb +++ b/docs/trulens_eval/langchain_instrumentation.ipynb @@ -99,7 +99,7 @@ "outputs": [], "source": [ "from langchain import LLMChain\n", - "from langchain import PromptTemplate\n", + "from langchain.prompts import PromptTemplate\n", "from langchain.callbacks import AsyncIteratorCallbackHandler\n", "from langchain.chains import LLMChain\n", "from langchain.chat_models.openai import ChatOpenAI\n", diff --git a/trulens_eval/examples/expositional/frameworks/langchain/langchain_model_comparison.ipynb b/trulens_eval/examples/expositional/frameworks/langchain/langchain_model_comparison.ipynb index e8a893e4a..cb9ac82b6 100644 --- a/trulens_eval/examples/expositional/frameworks/langchain/langchain_model_comparison.ipynb +++ b/trulens_eval/examples/expositional/frameworks/langchain/langchain_model_comparison.ipynb @@ -50,7 +50,7 @@ "from langchain.llms import OpenAI\n", "from langchain.prompts import ChatPromptTemplate, PromptTemplate\n", "from langchain.prompts import HumanMessagePromptTemplate\n", - "from langchain import PromptTemplate\n", + "from langchain.prompts import PromptTemplate\n", "from langchain.llms import OpenAI\n", "from langchain import LLMChain\n", "\n", diff --git a/trulens_eval/examples/expositional/models/alpaca7b_local_llm.ipynb b/trulens_eval/examples/expositional/models/alpaca7b_local_llm.ipynb index d41142ef4..a9d188ee1 100644 --- a/trulens_eval/examples/expositional/models/alpaca7b_local_llm.ipynb +++ b/trulens_eval/examples/expositional/models/alpaca7b_local_llm.ipynb @@ -39,7 +39,8 @@ "source": [ "from transformers import LlamaTokenizer, LlamaForCausalLM, GenerationConfig, pipeline\n", "from langchain.llms import HuggingFacePipeline\n", - "from langchain import PromptTemplate, LLMChain\n", + "from langchain import LLMChain\n", + "from langchain.prompts import PromptTemplate\n", "import openai\n", "import torch\n", "from trulens_eval.schema import Select\n", diff --git a/trulens_eval/trulens_eval/database/sqlalchemy_db.py b/trulens_eval/trulens_eval/database/sqlalchemy_db.py index 628fcc0cb..cd7cdce93 100644 --- a/trulens_eval/trulens_eval/database/sqlalchemy_db.py +++ b/trulens_eval/trulens_eval/database/sqlalchemy_db.py @@ -11,6 +11,7 @@ from sqlalchemy import create_engine from sqlalchemy import Engine from sqlalchemy import select +from sqlalchemy.schema import MetaData from sqlalchemy.orm import sessionmaker from trulens_eval import schema @@ -47,7 +48,11 @@ @for_all_methods( run_before(lambda self, *args, **kwargs: check_db_revision(self.engine)), - _except=["migrate_database", "reload_engine"] + _except=[ + "migrate_database", + "reload_engine", + "reset_database" # migrates database automatically + ] ) class SqlAlchemyDB(DB): engine_params: dict = Field(default_factory=dict) @@ -132,14 +137,11 @@ def migrate_database(self): logger.info("Your database does not need migration.") def reset_database(self): - deleted = 0 - with self.Session.begin() as session: - deleted += session.query(AppDefinition).delete() - deleted += session.query(FeedbackDefinition).delete() - deleted += session.query(Record).delete() - deleted += session.query(FeedbackResult).delete() + meta = MetaData() + meta.reflect(bind=self.engine) + meta.drop_all(bind=self.engine) - logger.info(f"Deleted {deleted} rows.") + self.migrate_database() def insert_record(self, record: schema.Record) -> schema.RecordID: # TODO: thread safety diff --git a/trulens_eval/trulens_eval/db_migration.py b/trulens_eval/trulens_eval/db_migration.py index 743196e1b..fc90fd27f 100644 --- a/trulens_eval/trulens_eval/db_migration.py +++ b/trulens_eval/trulens_eval/db_migration.py @@ -523,8 +523,6 @@ def _check_needs_migration(version: str, warn=False) -> None: """ compat_version = _get_compatibility_version(version) - print("compat_version=", compat_version) - if migration_versions.index(compat_version) > 0: if _upgrade_possible(compat_version): @@ -629,13 +627,16 @@ def migrate(db) -> None: Args: db (DB): the db object """ - # NOTE TO DEVELOPER: If this method fails: It's likely you made a db breaking change. - # Follow these steps to add a compatibility change + # NOTE TO DEVELOPER: If this method fails: It's likely you made a db + # breaking change. Follow these steps to add a compatibility change # - Update the __init__ version to the next one (if not already) # - In this file: add that version to `migration_versions` variable` - # - Add the migration step in `upgrade_paths` of the form `from_version`:(`to_version_you_just_created`, `migration_function`) - # - AFTER YOU PASS TESTS - add your newest db into `release_dbs//default.sqlite` - # - This is created by running the all_tools and llama_quickstart from a fresh db (you can `rm -rf` the sqlite file ) + # - Add the migration step in `upgrade_paths` of the form + # `from_version`:(`to_version_you_just_created`, `migration_function`) + # - AFTER YOU PASS TESTS - add your newest db into + # `release_dbs//default.sqlite` + # - This is created by running the all_tools and llama_quickstart from a + # fresh db (you can `rm -rf` the sqlite file ) # - TODO: automate this step original_db_file = db.filename global saved_db_locations diff --git a/trulens_eval/trulens_eval/feedback/groundedness.py b/trulens_eval/trulens_eval/feedback/groundedness.py index 8585546ec..de53ccbd7 100644 --- a/trulens_eval/trulens_eval/feedback/groundedness.py +++ b/trulens_eval/trulens_eval/feedback/groundedness.py @@ -1,5 +1,5 @@ import logging -from typing import Dict, List +from typing import Dict, List, Optional import numpy as np from tqdm.auto import tqdm @@ -23,7 +23,7 @@ class Groundedness(SerialModel, WithClassInfo): """ groundedness_provider: Provider - def __init__(self, groundedness_provider: Provider = None): + def __init__(self, groundedness_provider: Optional[Provider] = None, **kwargs): """Instantiates the groundedness providers. Currently the groundedness functions work well with a summarizer. This class will use an LLM to find the relevant strings in a text. The groundedness_provider can either be an LLM provider (such as OpenAI) or NLI with huggingface. @@ -53,7 +53,8 @@ def __init__(self, groundedness_provider: Provider = None): groundedness_provider = OpenAI() super().__init__( groundedness_provider=groundedness_provider, - obj=self # for WithClassInfo + obj=self, # for WithClassInfo + **kwargs ) def groundedness_measure(self, source: str, statement: str) -> float: diff --git a/trulens_eval/trulens_eval/feedback/groundtruth.py b/trulens_eval/trulens_eval/feedback/groundtruth.py index 69a9c177f..2c5e68865 100644 --- a/trulens_eval/trulens_eval/feedback/groundtruth.py +++ b/trulens_eval/trulens_eval/feedback/groundtruth.py @@ -33,7 +33,7 @@ class GroundTruthAgreement(SerialModel, WithClassInfo): # It's a class member because creating it is expensive bert_scorer: object - ground_truth_imp: Optional[Callable] = pydantic.Field(exclude=True) + ground_truth_imp: Optional[Callable] = pydantic.Field(None, exclude=True) model_config: ClassVar[dict] = dict( arbitrary_types_allowed = True @@ -43,7 +43,8 @@ def __init__( self, ground_truth: Union[List, Callable, FunctionOrMethod], provider: Optional[Provider] = None, - bert_scorer: Optional["BERTScorer"] = None + bert_scorer: Optional["BERTScorer"] = None, + **kwargs ): """Measures Agreement against a Ground Truth. @@ -94,7 +95,8 @@ def __init__( ground_truth_imp=ground_truth_imp, provider=provider, bert_scorer=bert_scorer, - obj=self # for WithClassInfo + obj=self, # for WithClassInfo + **kwargs ) def _find_response(self, prompt: str) -> Optional[str]: diff --git a/trulens_eval/trulens_eval/feedback/v2/feedback.py b/trulens_eval/trulens_eval/feedback/v2/feedback.py index a0d98c0c5..0052998fc 100644 --- a/trulens_eval/trulens_eval/feedback/v2/feedback.py +++ b/trulens_eval/trulens_eval/feedback/v2/feedback.py @@ -1,7 +1,7 @@ from abc import abstractmethod from typing import ClassVar, List, Optional -from langchain import PromptTemplate +from langchain.prompts import PromptTemplate from langchain.evaluation.criteria.eval_chain import _SUPPORTED_CRITERIA import pydantic diff --git a/trulens_eval/trulens_eval/utils/generated.py b/trulens_eval/trulens_eval/utils/generated.py index 17601f0a6..f61138d6c 100644 --- a/trulens_eval/trulens_eval/utils/generated.py +++ b/trulens_eval/trulens_eval/utils/generated.py @@ -17,7 +17,7 @@ def re_0_10_rating(str_val): matches = pat_0_10.fullmatch(str_val) if not matches: # Try soft match - matches = re.search('([0-9]+)(?=\D*$)', str_val) + matches = re.search(r'([0-9]+)(?=\D*$)', str_val) if not matches: logger.warning(f"0-10 rating regex failed to match on: '{str_val}'") return -10 # so this will be reported as -1 after division by 10 diff --git a/trulens_eval/trulens_eval/utils/pyschema.py b/trulens_eval/trulens_eval/utils/pyschema.py index b6a0425e7..97ac48acf 100644 --- a/trulens_eval/trulens_eval/utils/pyschema.py +++ b/trulens_eval/trulens_eval/utils/pyschema.py @@ -392,8 +392,17 @@ def load(self) -> object: else: extra_kwargs = {} - bindings = self.init_bindings.load(sig, extra_kwargs=extra_kwargs) - + try: + bindings = self.init_bindings.load(sig, extra_kwargs=extra_kwargs) + + except Exception as e: + msg = f"Error binding constructor args for object:\n" + msg += str(e) + "\n" + msg += f"\tobj={self}\n" + msg += f"\targs={self.init_bindings.args}\n" + msg += f"\tkwargs={self.init_bindings.kwargs}\n" + raise type(e)(msg) + return cls(*bindings.args, **bindings.kwargs)