DBConnector, TruSession, and App versioning changes (#1349)

* add base workspace code

* add trulens-workspaces-snowflake

* small docs changes

* rename to DefaultWorkspace

* add alembic revisions

* rename app_id to app_name and app_version

* Tru -> TruSession

* workspace -> connector

* app_id = hash of version and name

* update session.db reference

* alias for Tru

* throw deprecation warning and pass to DefaultDBConnector

* app_name / id deprecation

* remove workspace mentions

* update migration revision ids

* pr fixes

* validate schema name

* add migrations

* cleanup

* update imports

* populate with hash app_id

* more references

* docs fixes

* revert instance attr

* update session names in docs/variable naming

.pre-commit-config.yaml

@@ -70,3 +70,6 @@ repos:
       - id: poetry-check
         name: trulens-providers-openai-poetry-check
         args: [-C src/providers/openai]
+      - id: poetry-check
+        name: trulens-connectors-snowflake-poetry-check
+        args: [-C src/connectors/snowflake]

DEPRECATION.md

@@ -20,7 +20,7 @@
 - Database interfaces changed from sqlite to sqlalchemy. Sqlite databases are
   supported under the sqlaclchemy interface and other databases such as mysql
   and postgresql are also now usable. Running the migration scripts via
-  `Tru().migrate_database()` may be necessary.
+  `TruSession().migrate_database()` may be necessary.
 
 ## Changes in 0.7.0
 

docs/gen_ref_pages.py

@@ -88,7 +88,12 @@ def write_to_gen_files(
 instrument_packages = [
     f"instrument/{pkg_dir}" for pkg_dir in next(os.walk("src/instrument"))[1]
 ]
-packages = core_packages + provider_packages + instrument_packages
+connector_packages = [
+    f"connectors/{pkg_dir}" for pkg_dir in next(os.walk("src/connectors"))[1]
+]
+packages = (
+    core_packages + provider_packages + instrument_packages + connector_packages
+)
 print("Collecting from packages:", packages)
 
 # Write Index Page

docs/trulens/contributing/migration.md

@@ -10,7 +10,7 @@ If upgrading DB, You must do this step!!
 1. Make desired changes to SQLAlchemy orm models in `src/core/trulens/core/database/orm.py`.
 1. Get a database with the new changes:
    1. `rm default.sqlite`
-   1. Run `Tru()` to create a fresh database that uses the new ORM.
+   1. Run `TruSession()` to create a fresh database that uses the new ORM.
 1. Run automatic alembic revision script generator. This will generate a new python script in `src/core/trulens/core/database/migrations`.
    1. `cd src/core/trulens/core/database/migrations`
    1. `SQLALCHEMY_URL="sqlite:///../../../../../../default.sqlite" alembic revision --autogenerate -m "<short_description>" --rev-id "<next_integer_version>"`

docs/trulens/evaluation/feedback_implementations/custom_feedback_functions.ipynb

@@ -36,7 +36,7 @@
     "from trulens.core import Feedback\n",
     "from trulens.core import Provider\n",
     "from trulens.core import Select\n",
-    "from trulens.core import Tru\n",
+    "from trulens.core import TruSession\n",
     "\n",
     "\n",
     "class StandAlone(Provider):\n",
@@ -91,7 +91,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "tru = Tru()\n",
+    "tru = TruSession()\n",
     "feedback_results = tru.run_feedback_functions(\n",
     "    record=record, feedback_functions=[f_custom_function]\n",
     ")\n",

docs/trulens/evaluation/feedback_selectors/selecting_components.md

@@ -87,7 +87,9 @@ response = my_llm_app(query)
 from trulens.instrument.langchain import TruChain
 tru_recorder = TruChain(
     my_llm_app,
-    app_id='Chain1_ChatApplication')
+    app_name='ChatApplication',
+    app_version="Chain1",
+)
 
 response, tru_record = tru_recorder.with_record(my_llm_app, query)
 json_like = tru_record.layout_calls_as_app()
@@ -142,46 +144,13 @@ The top level JSON attributes are defined by the class structures.
 
 For a Record:
 
-```python
-class Record(SerialModel):
-    record_id: RecordID
-    app_id: AppID
-
-    cost: Optional[Cost] = None
-    perf: Optional[Perf] = None
-
-    ts: datetime = pydantic.Field(default_factory=lambda: datetime.now())
-
-    tags: str = ""
+::: trulens.core.schema.Record
 
-    main_input: Optional[JSON] = None
-    main_output: Optional[JSON] = None  # if no error
-    main_error: Optional[JSON] = None  # if error
-
-    # The collection of calls recorded. Note that these can be converted into a
-    # json structure with the same paths as the app that generated this record
-    # via `layout_calls_as_app`.
-    calls: Sequence[RecordAppCall] = []
-```
 
 For an App:
 
-```python
-class AppDefinition(WithClassInfo, SerialModel, ABC):
-    ...
+::: trulens.core.schema.AppDefinition
 
-    app_id: AppID
-
-    feedback_definitions: Sequence[FeedbackDefinition] = []
-
-    feedback_mode: FeedbackMode = FeedbackMode.WITH_APP_THREAD
-
-    root_class: Class
-
-    root_callable: ClassVar[FunctionOrMethod]
-
-    app: JSON
-```
 
 For your app, you can inspect the JSON-like structure by using the `dict`
 method:

docs/trulens/evaluation/running_feedback_functions/existing_data.md

@@ -14,7 +14,7 @@ feedback_result = provider.relevance("<some prompt>", "<some response>")
 In the case that you have already logged a run of your application with TruLens and have the record available, the process for running an (additional) evaluation on that record is by using `tru.run_feedback_functions`:
 
 ```python
-tru_rag = TruCustomApp(rag, app_id = 'RAG v1')
+tru_rag = TruCustomApp(rag, app_name="RAG", app_version="v1")
 
 result, record = tru_rag.with_record(rag.query, "How many professors are at UW in Seattle?")
 feedback_results = tru.run_feedback_functions(record, feedbacks=[f_lang_match, f_qa_relevance, f_context_relevance])
@@ -155,7 +155,7 @@ Then, the feedback functions can be passed to `TruVirtual` to construct the `rec
 from trulens.core.app.virtual import TruVirtual
 
 virtual_recorder = TruVirtual(
-    app_id="a virtual app",
+    app_name="a virtual app",
     app=virtual_app,
     feedbacks=[f_context_relevance]
 )

docs/trulens/evaluation/running_feedback_functions/with_app.md

@@ -24,7 +24,8 @@ from trulens.instrument.langchain import TruChain
 # f_lang_match, f_qa_relevance, f_context_relevance are feedback functions
 tru_recorder = TruChain(
     chain,
-    app_id='Chain1_ChatApplication',
+    app_name='ChatApplication',
+    app_version="Chain1",
     feedbacks=[f_lang_match, f_qa_relevance, f_context_relevance])
 ```
 
@@ -47,7 +48,8 @@ from trulens.core import FeedbackMode
 
 tru_recorder = TruChain(
     chain,
-    app_id='Chain1_ChatApplication',
+    app_name='ChatApplication',
+    app_version="Chain1",
     feedbacks=[f_lang_match, f_qa_relevance, f_context_relevance],
     feedback_mode=FeedbackMode.DEFERRED
     )

docs/trulens/getting_started/dashboard/index.md

@@ -4,13 +4,13 @@ TruLens provides a broad set of capabilities for evaluating and tracking applica
 
 ## TruLens Dashboard
 
-To view and examine application logs and feedback results, TruLens provides a built-in Streamlit dashboard. That app has two pages, the Leaderboard which displays aggregate feedback results and metadata for each application version, and the Evaluations page where you can more closely examine individual traces and feedback results. This dashboard is launched by [run_dashboard][trulens.dashboard.run_dashboard], and will run from a database url you specify with  [Tru()][trulens.core.Tru].
+To view and examine application logs and feedback results, TruLens provides a built-in Streamlit dashboard. That app has two pages, the Leaderboard which displays aggregate feedback results and metadata for each application version, and the Evaluations page where you can more closely examine individual traces and feedback results. This dashboard is launched by [run_dashboard][trulens.dashboard.run_dashboard], and will run from a database url you specify with  [TruSession()][trulens.core.TruSession].
 
 !!! example "Launch the TruLens dashboard"
 
     ```python
     from trulens.dashboard import run_dashboard
-    tru = Tru(database_url = ...) # or default.sqlite by default
+    tru = TruSession(database_url = ...) # or default.sqlite by default
     run_dashboard(tru)
     ```
 
@@ -41,12 +41,12 @@ Consider the below `app.py` which consists of a simple RAG application that is a
     ```python
 
     import streamlit as st
-    from trulens.core import Tru
+    from trulens.core import TruSession
 
     from base import rag # a rag app with a query method
     from base import tru_rag # a rag app wrapped by trulens
 
-    tru = Tru()
+    tru = TruSession()
 
     def generate_and_log_response(input_text):
         with tru_rag as recording:

docs/trulens/guides/trulens_eval_migration.md

@@ -34,15 +34,15 @@ As a result of these changes, the package structure for the TruLens varies from
 
 | TruLens Eval | TruLens | Additional Dependencies |
 |------------|-------------|------------------|
-| `trulens_eval.Tru` | [`trulens.core.Tru`][trulens.core.Tru] | |
+| `trulens_eval.Tru` | [`trulens.core.TruSession`][trulens.core.TruSession] | |
 | `trulens_eval.Feedback` | [`trulens.core.Feedback`][trulens.core.Feedback] | |
 | `trulens_eval.Select` | [`trulens.core.Select`][trulens.core.Select] | |
-| `trulens_eval.TruCustomApp`, `Tru().Custom(...)` | [`trulens.core.TruCustomApp`][trulens.core.TruCustomApp] | |
-| `trulens_eval.TruChain`, `Tru().Chain(...)` | [`trulens.instrument.langchain.TruChain`][trulens.instrument.langchain.TruChain] | `trulens-instrument-langchain` |
-| `trulens_eval.TruLlama`, `Tru().Llama(...)` | [`trulens.instrument.llamaindex.TruLlama`][trulens.instrument.llamaindex.TruLlama] | `trulens-instrument-llamaindex` |
+| `trulens_eval.TruCustomApp`, `TruSession().Custom(...)` | [`trulens.core.TruCustomApp`][trulens.core.TruCustomApp] | |
+| `trulens_eval.TruChain`, `TruSession().Chain(...)` | [`trulens.instrument.langchain.TruChain`][trulens.instrument.langchain.TruChain] | `trulens-instrument-langchain` |
+| `trulens_eval.TruLlama`, `TruSession().Llama(...)` | [`trulens.instrument.llamaindex.TruLlama`][trulens.instrument.llamaindex.TruLlama] | `trulens-instrument-llamaindex` |
 | `trulens_eval.OpenAI` | [`trulens.providers.openai.OpenAI`][trulens.providers.openai.OpenAI] | `trulens-providers-openai` |
 | `trulens_eval.Huggingface` | [`trulens.providers.huggingface.Huggingface`][trulens.providers.huggingface.Huggingface] | `trulens-providers-huggingface` |
 | `trulens_eval.guardrails.llama` | [`trulens.instrument.llamaindex.guardrails`][trulens.instrument.llamaindex.guardrails] | `trulens-instrument-llamaindex` |
-| `Tru().run_dashboard()` | [`trulens.dashboard.run_dashboard()`][trulens.dashboard.run_dashboard] | `trulens-dashboard` |
+| `TruSession().run_dashboard()` | [`trulens.dashboard.run_dashboard()`][trulens.dashboard.run_dashboard] | `trulens-dashboard` |
 
 To find a specific definition, use the search functionality or go directly to the [API Reference](../../reference/trulens/core/index.md).

docs/trulens/tracking/logging/logging.ipynb

@@ -29,13 +29,13 @@
     "from langchain.prompts import PromptTemplate\n",
     "from langchain_community.llms import OpenAI\n",
     "from trulens.core import Feedback\n",
-    "from trulens.core import Tru\n",
+    "from trulens.core import TruSession\n",
     "from trulens.instrument.langchain import TruChain\n",
     "from trulens.providers.huggingface import Huggingface\n",
     "\n",
-    "tru = Tru()\n",
+    "tru = TruSession()\n",
     "\n",
-    "Tru().migrate_database()\n",
+    "TruSession().migrate_database()\n",
     "\n",
     "full_prompt = HumanMessagePromptTemplate(\n",
     "    prompt=PromptTemplate(\n",
@@ -50,7 +50,9 @@
     "\n",
     "chain = LLMChain(llm=llm, prompt=chat_prompt_template, verbose=True)\n",
     "\n",
-    "truchain = TruChain(chain, app_id=\"Chain1_ChatApplication\", tru=tru)\n",
+    "truchain = TruChain(\n",
+    "    chain, app_name=\"ChatApplication\", app_version=\"Chain1\", tru=tru\n",
+    ")\n",
     "with truchain:\n",
     "    chain(\"This will be automatically logged.\")"
    ]
@@ -89,7 +91,8 @@
    "source": [
     "truchain = TruChain(\n",
     "    chain,\n",
-    "    app_id=\"Chain1_ChatApplication\",\n",
+    "    app_name=\"ChatApplication\",\n",
+    "    app_version=\"Chain1\",\n",
     "    feedbacks=[f_lang_match],  # feedback functions\n",
     "    tru=tru,\n",
     ")\n",
@@ -114,7 +117,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "tc = TruChain(chain, app_id=\"Chain1_ChatApplication\")"
+    "tc = TruChain(chain, app_name=\"ChatApplication\", app_version=\"Chain2\")"
    ]
   },
   {
@@ -273,7 +276,8 @@
    "source": [
     "truchain: TruChain = TruChain(\n",
     "    chain,\n",
-    "    app_id=\"Chain1_ChatApplication\",\n",
+    "    app_name=\"ChatApplication\",\n",
+    "    app_version=\"chain_1\",\n",
     "    feedbacks=[f_lang_match],\n",
     "    tru=tru,\n",
     "    feedback_mode=\"deferred\",\n",

docs/trulens/tracking/logging/where_to_log/index.md

@@ -8,8 +8,8 @@ See [this article](https://docs.sqlalchemy.org/en/20/core/engines.html#database-
 For example, for Postgres database `trulens` running on `localhost` with username `trulensuser` and password `password` set up a connection like so.
 
 ```
-from trulens.core import Tru
-tru = Tru(database_url="postgresql://trulensuser:password@localhost/trulens")
+from trulens.core import TruSession
+tru = TruSession(database_url="postgresql://trulensuser:password@localhost/trulens")
 ```
 
 After which you should receive the following message:

docs/trulens/tracking/logging/where_to_log/log_in_snowflake.md

@@ -27,8 +27,8 @@ Connecting TruLens to a Snowflake database for logging traces and evaluations on
 !!! example "Connect TruLens to your Snowflake database"
 
     ```python
-    from trulens.core import Tru
-    tru = Tru(
+    from trulens.core import TruSession
+    tru = TruSession(
         name="MyApp",
         snowflake_connection_parameters={
             account: "<account>",

examples/dev/dummy_app/agent.py

@@ -47,7 +47,7 @@ def __init__(
         self.app = app
 
         if self.use_app:
-            self.tru_app = TruCustomApp(self.app, app_id=description)
+            self.tru_app = TruCustomApp(self.app, app_name=description)
 
         self.dummy_allocate()
 

examples/dev/dummy_app/dummy_app_example.ipynb

@@ -35,13 +35,13 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from trulens.core import Tru\n",
+    "from trulens.core import TruSession\n",
     "from trulens.core.app.custom import TruCustomApp\n",
     "\n",
     "from examples.dev.dummy_app.agent import DummyAgent\n",
     "from examples.dev.dummy_app.app import DummyApp\n",
     "\n",
-    "tru = Tru()\n",
+    "tru = TruSession()\n",
     "tru.reset_database()"
    ]
   },
@@ -57,7 +57,7 @@
     "# Create trulens wrapper:\n",
     "ta = TruCustomApp(\n",
     "    ca,\n",
-    "    app_id=\"customapp\",\n",
+    "    app_name=\"customapp\",\n",
     ")"
    ]
   },
@@ -177,8 +177,7 @@
     "        for use_recorder in [\n",
     "            False\n",
     "        ]:  # , True]: # currently not working with nested recorder\n",
-    "            app_id = f\"custom_app (use_parralel={use_parallel} use_app={use_app} use_recorder={use_recorder})\"\n",
-    "            print(app_id)\n",
+    "            app_version = f\"(use_parralel={use_parallel} use_app={use_app} use_recorder={use_recorder})\"\n",
     "\n",
     "            # Create custom app:\n",
     "            ca = DummyApp(\n",
@@ -194,7 +193,8 @@
     "            # Create trulens wrapper:\n",
     "            ta = TruCustomApp(\n",
     "                ca,\n",
-    "                app_id=app_id,\n",
+    "                app_name=\"customapp\",\n",
+    "                app_version=app_version,\n",
     "            )\n",
     "\n",
     "            # Sync usage\n",

examples/experimental/MultiQueryRetrievalLangchain.ipynb

@@ -60,7 +60,7 @@
     "from langchain_openai import OpenAIEmbeddings\n",
     "from langchain_text_splitters import RecursiveCharacterTextSplitter\n",
     "from trulens.core import Feedback\n",
-    "from trulens.core import Tru\n",
+    "from trulens.core import TruSession\n",
     "from trulens.instrument.langchain import TruChain\n",
     "from trulens.providers.openai import OpenAI"
    ]
@@ -142,7 +142,7 @@
    "source": [
     "import numpy as np\n",
     "\n",
-    "tru = Tru()\n",
+    "tru = TruSession()\n",
     "tru.reset_database()\n",
     "# Initialize provider class\n",
     "provider = OpenAI()\n",
@@ -188,7 +188,7 @@
     "\n",
     "tru_recorder = TruChain(\n",
     "    rag_chain,\n",
-    "    app_id=\"MultiReg\",\n",
+    "    app_name=\"MultiReg\",\n",
     "    feedbacks=[f_answer_relevance, f_context_relevance, f_groundedness],\n",
     ")\n",
     "\n",