diff --git a/docs/trulens_eval/api/trace/index.md b/docs/trulens_eval/api/trace/index.md new file mode 100644 index 000000000..d08dc68ce --- /dev/null +++ b/docs/trulens_eval/api/trace/index.md @@ -0,0 +1,3 @@ +# Trace + +::: trulens_eval.trace \ No newline at end of file diff --git a/docs/trulens_eval/api/trace/span.md b/docs/trulens_eval/api/trace/span.md new file mode 100644 index 000000000..547b5c77c --- /dev/null +++ b/docs/trulens_eval/api/trace/span.md @@ -0,0 +1,3 @@ +# Span + +::: trulens_eval.trace.span diff --git a/docs/trulens_eval/api/trace/tracer.md b/docs/trulens_eval/api/trace/tracer.md new file mode 100644 index 000000000..d8b2bef75 --- /dev/null +++ b/docs/trulens_eval/api/trace/tracer.md @@ -0,0 +1,4 @@ +# Tracer + +::: trulens_eval.trace.tracer + diff --git a/docs/trulens_eval/contributing/design.md b/docs/trulens_eval/contributing/design.md index 87be9037d..a141a9666 100644 --- a/docs/trulens_eval/contributing/design.md +++ b/docs/trulens_eval/contributing/design.md @@ -62,11 +62,11 @@ In addition to collecting app parameters, we also collect: - (subset of components) App class information: - - This allows us to deserialize some objects. Pydantic models can be - deserialized once we know their class and fields, for example. - - This information is also used to determine component types without having - to deserialize them first. - - See [Class][trulens_eval.utils.pyschema.Class] for details. + - This allows us to deserialize some objects. Pydantic models can be + deserialized once we know their class and fields, for example. + - This information is also used to determine component types without having + to deserialize them first. + - See [Class][trulens_eval.utils.pyschema.Class] for details. ### Functions/Methods @@ -158,7 +158,11 @@ our reliance on info stored on the stack. Therefore we have a limitation: [ThreadPoolExecutor][trulens_eval.utils.threading.ThreadPoolExecutor] also defined in `utils/threading.py` in order for instrumented methods called in a thread to be tracked. As we rely on call stack for call instrumentation we - need to preserve the stack before a thread start which python does not do. + need to preserve the stack before a thread start which python does not do. + +OpenTelemetry has similar problems and comes with similar solutions. See for + example [OpenTelemetry thread + instrumentation](https://opentelemetry-python-contrib.readthedocs.io/en/latest/instrumentation/threading/threading.html). #### Async diff --git a/docs/trulens_eval/contributing/techdebt.md b/docs/trulens_eval/contributing/techdebt.md index 107c48121..15a19fc63 100644 --- a/docs/trulens_eval/contributing/techdebt.md +++ b/docs/trulens_eval/contributing/techdebt.md @@ -32,6 +32,9 @@ See `instruments.py` docstring for discussion why these are done. object that implements `__call__` has been instrumented. Hacks to avoid warnings about lack of instrumentation. +- "HACK015" -- Add `__hash__` onto `opentelemetry.trace.span.SpanContext` by + changing their `__class__`. + ## Thread overriding See `instruments.py` docstring for discussion why these are done. diff --git a/mkdocs.yml b/mkdocs.yml index 92a6ab758..1bc5bd63a 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -67,6 +67,7 @@ plugins: - https://typing-extensions.readthedocs.io/en/latest/objects.inv - https://docs.llamaindex.ai/en/stable/objects.inv - https://docs.sqlalchemy.org/en/20/objects.inv + - https://opentelemetry-python.readthedocs.io/en/latest/objects.inv options: extensions: - pydantic: { schema: true } @@ -269,6 +270,10 @@ nav: - trulens_eval/api/utils/json.md - trulens_eval/api/utils/frameworks.md - trulens_eval/api/utils/utils.md + - "Experimental: Tracing": + - trulens_eval/api/trace/index.md + - trulens_eval/api/trace/span.md + - trulens_eval/api/trace/tracer.md - 🤝 Contributing: - trulens_eval/contributing/index.md - 🧭 Design: trulens_eval/contributing/design.md diff --git a/trulens_eval/examples/experimental/dev_notebook.ipynb b/trulens_eval/examples/experimental/dev_notebook.ipynb index c7890d345..81b6e0478 100644 --- a/trulens_eval/examples/experimental/dev_notebook.ipynb +++ b/trulens_eval/examples/experimental/dev_notebook.ipynb @@ -70,53 +70,6 @@ "# tru.db.migrate_database()" ] }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# tru.db.migrate_database()\n", - "tru.migrate_database()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "for t in tru.db.orm.registry.values():\n", - " print(t)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "from trulens_eval.database.utils import copy_database" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "tru.db" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "copy_database(\"sqlite:///default.sqlite\", \"sqlite:///default2.sqlite\", src_prefix=\"dev\", tgt_prefix=\"dev\")" - ] - }, { "cell_type": "code", "execution_count": null, diff --git a/trulens_eval/examples/experimental/dummy_example.ipynb b/trulens_eval/examples/experimental/dummy_example.ipynb index ea4399954..bc22350a4 100644 --- a/trulens_eval/examples/experimental/dummy_example.ipynb +++ b/trulens_eval/examples/experimental/dummy_example.ipynb @@ -50,7 +50,7 @@ "from trulens_eval import Feedback\n", "from trulens_eval import Tru\n", "from trulens_eval.feedback.provider.hugs import Dummy\n", - "from trulens_eval.schema import FeedbackMode\n", + "from trulens_eval.schema.feedback import FeedbackMode\n", "from trulens_eval.tru_custom_app import TruCustomApp\n", "from trulens_eval.utils.threading import TP\n", "\n", @@ -102,7 +102,7 @@ "ta = TruCustomApp(\n", " ca,\n", " app_id=\"customapp\",\n", - " feedbacks=[f_dummy1, f_dummy2, f_dummy3],\n", + " # feedbacks=[f_dummy1, f_dummy2, f_dummy3],\n", " feedback_mode=FeedbackMode.DEFERRED\n", ")" ] @@ -116,7 +116,7 @@ "# Sequential app invocation.\n", "\n", "if True:\n", - " for i in tqdm(range(128), desc=\"invoking app\"):\n", + " for i in tqdm(range(2), desc=\"invoking app\"):\n", " with ta as recorder:\n", " res = ca.respond_to_query(f\"hello {i}\")\n", "\n", @@ -229,7 +229,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.18" + "version": "3.11.6" }, "orig_nbformat": 4 }, diff --git a/trulens_eval/examples/experimental/spans_example.ipynb b/trulens_eval/examples/experimental/spans_example.ipynb new file mode 100644 index 000000000..ccecdc550 --- /dev/null +++ b/trulens_eval/examples/experimental/spans_example.ipynb @@ -0,0 +1,725 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Working with Spans" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "%load_ext autoreload\n", + "%autoreload 2\n", + "from pathlib import Path\n", + "import sys\n", + "\n", + "# If running from github repo, can use this:\n", + "repo = Path().cwd().parent.parent.resolve()\n", + "sys.path.append(str(repo))" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from pprint import pformat\n", + "from pprint import pprint\n", + "\n", + "from examples.expositional.end2end_apps.custom_app.custom_app import CustomApp\n", + "import pandas as pd\n", + "\n", + "from trulens_eval import instruments\n", + "from trulens_eval.trace.category import Categorizer\n", + "from trulens_eval.tru_custom_app import TruCustomApp" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "🦑 Tru initialized with db url sqlite:///default.sqlite .\n", + "🛑 Secret keys may be written to the database. See the `database_redact_keys` option of Tru` to prevent this.\n" + ] + } + ], + "source": [ + "# Create custom app:\n", + "ca = CustomApp(delay=0.0, alloc=0)\n", + "\n", + "# Create trulens wrapper:\n", + "ta = TruCustomApp(\n", + " ca,\n", + " app_id=\"customapp\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Module examples.expositional.end2end_apps.custom_app.custom_app*\n", + " Class examples.expositional.end2end_apps.custom_app.custom_app.CustomApp\n", + " Method retrieve_chunks: (self, data)\n", + " Method respond_to_query: (self, input)\n", + " Method arespond_to_query: (self, input)\n", + " Class examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate\n", + " Method fill: (self, question, answer)\n", + "\n", + "Module examples.expositional.end2end_apps.custom_app.custom_llm*\n", + " Class examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM\n", + " Method generate: (self, prompt: str)\n", + "\n", + "Module examples.expositional.end2end_apps.custom_app.custom_memory*\n", + " Class examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory\n", + " Method remember: (self, data: str)\n", + "\n", + "Module examples.expositional.end2end_apps.custom_app.custom_retriever*\n", + " Class examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever\n", + " Method retrieve_chunks: (self, data)\n", + " Span type: SpanType.RETRIEVER\n", + "\n", + "Module trulens_eval.*\n", + " Class trulens_eval.feedback.feedback.Feedback\n", + " Method __call__: (self, *args, **kwargs) -> 'Any'\n", + "\n" + ] + } + ], + "source": [ + "instruments.Instrument().print_instrumentation()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "with ta as recorder:\n", + " res = ca.respond_to_query(f\"hello\")\n", + "\n", + "rec = recorder.get()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "{'stack': [{'path': 'app',\n", + " 'method': {'obj': {'cls': {'name': 'CustomApp',\n", + " 'module': {'package_name': 'examples.expositional.end2end_apps.custom_app',\n", + " 'module_name': 'examples.expositional.end2end_apps.custom_app.custom_app'},\n", + " 'bases': None},\n", + " 'id': 11462560208,\n", + " 'init_bindings': None},\n", + " 'name': 'respond_to_query'}},\n", + " {'path': 'app',\n", + " 'method': {'obj': {'cls': {'name': 'CustomApp',\n", + " 'module': {'package_name': 'examples.expositional.end2end_apps.custom_app',\n", + " 'module_name': 'examples.expositional.end2end_apps.custom_app.custom_app'},\n", + " 'bases': None},\n", + " 'id': 11462560208,\n", + " 'init_bindings': None},\n", + " 'name': 'retrieve_chunks'}},\n", + " {'path': 'app.retriever',\n", + " 'method': {'obj': {'cls': {'name': 'CustomRetriever',\n", + " 'module': {'package_name': 'examples.expositional.end2end_apps.custom_app',\n", + " 'module_name': 'examples.expositional.end2end_apps.custom_app.custom_retriever'},\n", + " 'bases': None},\n", + " 'id': 11476781776,\n", + " 'init_bindings': None},\n", + " 'name': 'retrieve_chunks'}}],\n", + " 'args': {'data': 'hello'},\n", + " 'rets': ['Relevant chunk: HELLO',\n", + " 'Relevant chunk: olleh',\n", + " \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"],\n", + " 'error': None,\n", + " 'perf': {'start_time': '2024-04-26T18:32:06.983465',\n", + " 'end_time': '2024-04-26T18:32:07.007769'},\n", + " 'pid': 71410,\n", + " 'tid': 15311948}" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "rec.calls[0].model_dump()" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
012345
0340282366920938463461285262137769617781rootSpanRoot4370756423165289941NaN{'trulens_eval@span_type': 'SpanRoot', 'trulen...
1340282366920938463461285262137769617781retrieve_chunksSpanRetriever41547318414298009869.875346e+18{'trulens_eval@span_type': 'SpanRetriever', 't...
2340282366920938463461285262137769617781retrieve_chunksSpanOther98753459883409037413.375420e+18{'trulens_eval@span_type': 'SpanOther', 'trule...
3340282366920938463461285262137769617781rememberSpanOther135084943061387644913.375420e+18{'trulens_eval@span_type': 'SpanOther', 'trule...
4340282366920938463461285262137769617781generateSpanOther2676102278785818833.375420e+18{'trulens_eval@span_type': 'SpanOther', 'trule...
5340282366920938463461285262137769617781fillSpanOther59871486530282570663.375420e+18{'trulens_eval@span_type': 'SpanOther', 'trule...
6340282366920938463461285262137769617781rememberSpanOther181912051972634624023.375420e+18{'trulens_eval@span_type': 'SpanOther', 'trule...
7340282366920938463461285262137769617781respond_to_querySpanOther33754204971678570374.370756e+18{'trulens_eval@span_type': 'SpanOther', 'trule...
\n", + "
" + ], + "text/plain": [ + " 0 1 2 \\\n", + "0 340282366920938463461285262137769617781 root SpanRoot \n", + "1 340282366920938463461285262137769617781 retrieve_chunks SpanRetriever \n", + "2 340282366920938463461285262137769617781 retrieve_chunks SpanOther \n", + "3 340282366920938463461285262137769617781 remember SpanOther \n", + "4 340282366920938463461285262137769617781 generate SpanOther \n", + "5 340282366920938463461285262137769617781 fill SpanOther \n", + "6 340282366920938463461285262137769617781 remember SpanOther \n", + "7 340282366920938463461285262137769617781 respond_to_query SpanOther \n", + "\n", + " 3 4 \\\n", + "0 4370756423165289941 NaN \n", + "1 4154731841429800986 9.875346e+18 \n", + "2 9875345988340903741 3.375420e+18 \n", + "3 13508494306138764491 3.375420e+18 \n", + "4 267610227878581883 3.375420e+18 \n", + "5 5987148653028257066 3.375420e+18 \n", + "6 18191205197263462402 3.375420e+18 \n", + "7 3375420497167857037 4.370756e+18 \n", + "\n", + " 5 \n", + "0 {'trulens_eval@span_type': 'SpanRoot', 'trulen... \n", + "1 {'trulens_eval@span_type': 'SpanRetriever', 't... \n", + "2 {'trulens_eval@span_type': 'SpanOther', 'trule... \n", + "3 {'trulens_eval@span_type': 'SpanOther', 'trule... \n", + "4 {'trulens_eval@span_type': 'SpanOther', 'trule... \n", + "5 {'trulens_eval@span_type': 'SpanOther', 'trule... \n", + "6 {'trulens_eval@span_type': 'SpanOther', 'trule... \n", + "7 {'trulens_eval@span_type': 'SpanOther', 'trule... " + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "spans = Categorizer.spans_of_record(rec)\n", + "\n", + "pd.DataFrame([(s.trace_id, s.name, s.span_type, s.span_id, s.parent_span_id, s.attributes) for s in spans])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "SpanRoot(name='root', kind=, status=, status_description=None, start_timestamp=1714181527594077000, end_timestamp=1714156327458285000, context=HashableSpanContext(trace_id=0xffffffffffffffffe301278662146975, span_id=0x3ca80bcf428625d5, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={}, attributes={'trulens_eval@span_type': 'SpanRoot', 'trulens_eval@record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac'}, attributes_metadata={}, record=Record(record_id='record_hash_b9a1b716f15720549fa790baca2f08ac', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), ts=datetime.datetime(2024, 4, 26, 18, 32, 7, 458338), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11476781776, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 983465), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7769)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 889975), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7875)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 156163), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 180696)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11476641424, init_bindings=None), name='generate'))], args={'prompt': \"Relevant chunk: HELLO processed,Relevant chunk: olleh processed,Relevant chunk: I allocated 56 bytes to pretend I'm doing something. processed\"}, rets=\"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 255179), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 280144)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11484672336, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\"}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 343782), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 369505)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 431826), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458224)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), pid=71410, tid=15311948)], feedback_and_future_results=[], feedback_results=[]), tags=[], span_type='SpanRoot', record_id='record_hash_b9a1b716f15720549fa790baca2f08ac')\n", + "{'attributes': {'trulens_eval@record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac',\n", + " 'trulens_eval@span_type': 'SpanRoot'},\n", + " 'attributes_metadata': {},\n", + " 'context': (340282366920938463461285262137769617781,\n", + " 4370756423165289941,\n", + " False,\n", + " 0,\n", + " [],\n", + " True),\n", + " 'end_timestamp': 1714156327458285000,\n", + " 'events': [],\n", + " 'kind': ,\n", + " 'links': [],\n", + " 'name': 'root',\n", + " 'record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac',\n", + " 'span_type': 'SpanRoot',\n", + " 'start_timestamp': 1714181527594077000,\n", + " 'status': ,\n", + " 'status_description': None,\n", + " 'tags': []}\n", + "\n", + "SpanRetriever(name='retrieve_chunks', kind=, status=, status_description=None, start_timestamp=1714156326983465000, end_timestamp=1714156327007769000, context=HashableSpanContext(trace_id=0xffffffffffffffffe301278662146975, span_id=0x39a89298d96d881a, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0xffffffffffffffffe301278662146975, span_id=0x890c46dac5236f3d, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanRetriever', 'trulens_eval@input_text': 'hello', 'trulens_eval@retrieved_contexts': ['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], 'trulens_eval@record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac', 'trulens_eval@inputs': {'data': 'hello'}, 'trulens_eval@output': ['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_b9a1b716f15720549fa790baca2f08ac', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), ts=datetime.datetime(2024, 4, 26, 18, 32, 7, 458338), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11476781776, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 983465), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7769)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 889975), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7875)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 156163), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 180696)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11476641424, init_bindings=None), name='generate'))], args={'prompt': \"Relevant chunk: HELLO processed,Relevant chunk: olleh processed,Relevant chunk: I allocated 56 bytes to pretend I'm doing something. processed\"}, rets=\"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 255179), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 280144)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11484672336, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\"}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 343782), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 369505)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 431826), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458224)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), pid=71410, tid=15311948)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11476781776, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 983465), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7769)), pid=71410, tid=15311948), tags=[], span_type='SpanRetriever', record_id='record_hash_b9a1b716f15720549fa790baca2f08ac', inputs={'data': 'hello'}, output=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, input_text='hello', input_embedding=None, distance_type=None, num_contexts=None, retrieved_contexts=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"])\n", + "{'attributes': {'trulens_eval@error': None,\n", + " 'trulens_eval@input_text': 'hello',\n", + " 'trulens_eval@inputs': {'data': 'hello'},\n", + " 'trulens_eval@output': ['Relevant chunk: HELLO',\n", + " 'Relevant chunk: olleh',\n", + " 'Relevant chunk: I allocated 56 bytes '\n", + " \"to pretend I'm doing something.\"],\n", + " 'trulens_eval@record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac',\n", + " 'trulens_eval@retrieved_contexts': ['Relevant chunk: HELLO',\n", + " 'Relevant chunk: olleh',\n", + " 'Relevant chunk: I '\n", + " 'allocated 56 bytes to '\n", + " \"pretend I'm doing \"\n", + " 'something.'],\n", + " 'trulens_eval@span_type': 'SpanRetriever'},\n", + " 'attributes_metadata': {},\n", + " 'context': (340282366920938463461285262137769617781,\n", + " 4154731841429800986,\n", + " False,\n", + " 0,\n", + " [],\n", + " True),\n", + " 'distance_type': None,\n", + " 'end_timestamp': 1714156327007769000,\n", + " 'error': None,\n", + " 'events': [],\n", + " 'input_embedding': None,\n", + " 'input_text': 'hello',\n", + " 'inputs': {'data': 'hello'},\n", + " 'kind': ,\n", + " 'links': [((340282366920938463461285262137769617781,\n", + " 9875345988340903741,\n", + " False,\n", + " 0,\n", + " [],\n", + " True),\n", + " {'trulens_eval@relationship': 'parent'})],\n", + " 'name': 'retrieve_chunks',\n", + " 'num_contexts': None,\n", + " 'output': ['Relevant chunk: HELLO',\n", + " 'Relevant chunk: olleh',\n", + " \"Relevant chunk: I allocated 56 bytes to pretend I'm doing \"\n", + " 'something.'],\n", + " 'record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac',\n", + " 'retrieved_contexts': ['Relevant chunk: HELLO',\n", + " 'Relevant chunk: olleh',\n", + " \"Relevant chunk: I allocated 56 bytes to pretend I'm \"\n", + " 'doing something.'],\n", + " 'span_type': 'SpanRetriever',\n", + " 'start_timestamp': 1714156326983465000,\n", + " 'status': ,\n", + " 'status_description': None,\n", + " 'tags': []}\n", + "\n", + "SpanOther(name='retrieve_chunks', kind=, status=, status_description=None, start_timestamp=1714156326889975000, end_timestamp=1714156327007875000, context=HashableSpanContext(trace_id=0xffffffffffffffffe301278662146975, span_id=0x890c46dac5236f3d, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0xffffffffffffffffe301278662146975, span_id=0x2ed7e70ef543358d, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanOther', 'trulens_eval@record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac', 'trulens_eval@inputs': {'data': 'hello'}, 'trulens_eval@output': ['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_b9a1b716f15720549fa790baca2f08ac', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), ts=datetime.datetime(2024, 4, 26, 18, 32, 7, 458338), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11476781776, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 983465), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7769)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 889975), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7875)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 156163), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 180696)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11476641424, init_bindings=None), name='generate'))], args={'prompt': \"Relevant chunk: HELLO processed,Relevant chunk: olleh processed,Relevant chunk: I allocated 56 bytes to pretend I'm doing something. processed\"}, rets=\"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 255179), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 280144)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11484672336, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\"}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 343782), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 369505)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 431826), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458224)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), pid=71410, tid=15311948)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 889975), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7875)), pid=71410, tid=15311948), tags=[], span_type='SpanOther', record_id='record_hash_b9a1b716f15720549fa790baca2f08ac', inputs={'data': 'hello'}, output=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None)\n", + "{'attributes': {'trulens_eval@error': None,\n", + " 'trulens_eval@inputs': {'data': 'hello'},\n", + " 'trulens_eval@output': ['Relevant chunk: HELLO',\n", + " 'Relevant chunk: olleh',\n", + " 'Relevant chunk: I allocated 56 bytes '\n", + " \"to pretend I'm doing something.\"],\n", + " 'trulens_eval@record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac',\n", + " 'trulens_eval@span_type': 'SpanOther'},\n", + " 'attributes_metadata': {},\n", + " 'context': (340282366920938463461285262137769617781,\n", + " 9875345988340903741,\n", + " False,\n", + " 0,\n", + " [],\n", + " True),\n", + " 'end_timestamp': 1714156327007875000,\n", + " 'error': None,\n", + " 'events': [],\n", + " 'inputs': {'data': 'hello'},\n", + " 'kind': ,\n", + " 'links': [((340282366920938463461285262137769617781,\n", + " 3375420497167857037,\n", + " False,\n", + " 0,\n", + " [],\n", + " True),\n", + " {'trulens_eval@relationship': 'parent'})],\n", + " 'name': 'retrieve_chunks',\n", + " 'output': ['Relevant chunk: HELLO',\n", + " 'Relevant chunk: olleh',\n", + " \"Relevant chunk: I allocated 56 bytes to pretend I'm doing \"\n", + " 'something.'],\n", + " 'record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac',\n", + " 'span_type': 'SpanOther',\n", + " 'start_timestamp': 1714156326889975000,\n", + " 'status': ,\n", + " 'status_description': None,\n", + " 'tags': []}\n", + "\n", + "SpanOther(name='remember', kind=, status=, status_description=None, start_timestamp=1714156327156163000, end_timestamp=1714156327180696000, context=HashableSpanContext(trace_id=0xffffffffffffffffe301278662146975, span_id=0xbb77d00017c9b4cb, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0xffffffffffffffffe301278662146975, span_id=0x2ed7e70ef543358d, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanOther', 'trulens_eval@record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac', 'trulens_eval@inputs': {'data': 'hello'}, 'trulens_eval@output': None, 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_b9a1b716f15720549fa790baca2f08ac', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), ts=datetime.datetime(2024, 4, 26, 18, 32, 7, 458338), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11476781776, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 983465), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7769)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 889975), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7875)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 156163), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 180696)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11476641424, init_bindings=None), name='generate'))], args={'prompt': \"Relevant chunk: HELLO processed,Relevant chunk: olleh processed,Relevant chunk: I allocated 56 bytes to pretend I'm doing something. processed\"}, rets=\"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 255179), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 280144)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11484672336, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\"}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 343782), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 369505)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 431826), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458224)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), pid=71410, tid=15311948)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 156163), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 180696)), pid=71410, tid=15311948), tags=[], span_type='SpanOther', record_id='record_hash_b9a1b716f15720549fa790baca2f08ac', inputs={'data': 'hello'}, output=None, error=None)\n", + "{'attributes': {'trulens_eval@error': None,\n", + " 'trulens_eval@inputs': {'data': 'hello'},\n", + " 'trulens_eval@output': None,\n", + " 'trulens_eval@record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac',\n", + " 'trulens_eval@span_type': 'SpanOther'},\n", + " 'attributes_metadata': {},\n", + " 'context': (340282366920938463461285262137769617781,\n", + " 13508494306138764491,\n", + " False,\n", + " 0,\n", + " [],\n", + " True),\n", + " 'end_timestamp': 1714156327180696000,\n", + " 'error': None,\n", + " 'events': [],\n", + " 'inputs': {'data': 'hello'},\n", + " 'kind': ,\n", + " 'links': [((340282366920938463461285262137769617781,\n", + " 3375420497167857037,\n", + " False,\n", + " 0,\n", + " [],\n", + " True),\n", + " {'trulens_eval@relationship': 'parent'})],\n", + " 'name': 'remember',\n", + " 'output': None,\n", + " 'record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac',\n", + " 'span_type': 'SpanOther',\n", + " 'start_timestamp': 1714156327156163000,\n", + " 'status': ,\n", + " 'status_description': None,\n", + " 'tags': []}\n", + "\n", + "SpanOther(name='generate', kind=, status=, status_description=None, start_timestamp=1714156327255179000, end_timestamp=1714156327280144000, context=HashableSpanContext(trace_id=0xffffffffffffffffe301278662146975, span_id=0x03b6be159af6d67b, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0xffffffffffffffffe301278662146975, span_id=0x2ed7e70ef543358d, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanOther', 'trulens_eval@record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac', 'trulens_eval@inputs': {'prompt': \"Relevant chunk: HELLO processed,Relevant chunk: olleh processed,Relevant chunk: I allocated 56 bytes to pretend I'm doing something. processed\"}, 'trulens_eval@output': \"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\", 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_b9a1b716f15720549fa790baca2f08ac', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), ts=datetime.datetime(2024, 4, 26, 18, 32, 7, 458338), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11476781776, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 983465), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7769)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 889975), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7875)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 156163), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 180696)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11476641424, init_bindings=None), name='generate'))], args={'prompt': \"Relevant chunk: HELLO processed,Relevant chunk: olleh processed,Relevant chunk: I allocated 56 bytes to pretend I'm doing something. processed\"}, rets=\"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 255179), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 280144)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11484672336, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\"}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 343782), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 369505)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 431826), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458224)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), pid=71410, tid=15311948)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11476641424, init_bindings=None), name='generate'))], args={'prompt': \"Relevant chunk: HELLO processed,Relevant chunk: olleh processed,Relevant chunk: I allocated 56 bytes to pretend I'm doing something. processed\"}, rets=\"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 255179), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 280144)), pid=71410, tid=15311948), tags=[], span_type='SpanOther', record_id='record_hash_b9a1b716f15720549fa790baca2f08ac', inputs={'prompt': \"Relevant chunk: HELLO processed,Relevant chunk: olleh processed,Relevant chunk: I allocated 56 bytes to pretend I'm doing something. processed\"}, output=\"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\", error=None)\n", + "{'attributes': {'trulens_eval@error': None,\n", + " 'trulens_eval@inputs': {'prompt': 'Relevant chunk: HELLO '\n", + " 'processed,Relevant chunk: '\n", + " 'olleh processed,Relevant '\n", + " 'chunk: I allocated 56 bytes '\n", + " \"to pretend I'm doing \"\n", + " 'something. processed'},\n", + " 'trulens_eval@output': \"herp dessecorp .gnihtemos gniod m'I \"\n", + " 'dneterp ot setyb 65 detacolla I :knuhc '\n", + " 'tnaveleR,dessecorp hello :knuhc '\n", + " 'tnaveleR,dessecorp OLLEH :knuhc '\n", + " 'tnaveleR derp and 56 bytes',\n", + " 'trulens_eval@record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac',\n", + " 'trulens_eval@span_type': 'SpanOther'},\n", + " 'attributes_metadata': {},\n", + " 'context': (340282366920938463461285262137769617781,\n", + " 267610227878581883,\n", + " False,\n", + " 0,\n", + " [],\n", + " True),\n", + " 'end_timestamp': 1714156327280144000,\n", + " 'error': None,\n", + " 'events': [],\n", + " 'inputs': {'prompt': 'Relevant chunk: HELLO processed,Relevant chunk: olleh '\n", + " 'processed,Relevant chunk: I allocated 56 bytes to '\n", + " \"pretend I'm doing something. processed\"},\n", + " 'kind': ,\n", + " 'links': [((340282366920938463461285262137769617781,\n", + " 3375420497167857037,\n", + " False,\n", + " 0,\n", + " [],\n", + " True),\n", + " {'trulens_eval@relationship': 'parent'})],\n", + " 'name': 'generate',\n", + " 'output': \"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla \"\n", + " 'I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH '\n", + " ':knuhc tnaveleR derp and 56 bytes',\n", + " 'record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac',\n", + " 'span_type': 'SpanOther',\n", + " 'start_timestamp': 1714156327255179000,\n", + " 'status': ,\n", + " 'status_description': None,\n", + " 'tags': []}\n", + "\n", + "SpanOther(name='fill', kind=, status=, status_description=None, start_timestamp=1714156327343782000, end_timestamp=1714156327369505000, context=HashableSpanContext(trace_id=0xffffffffffffffffe301278662146975, span_id=0x53169ffa89352d2a, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0xffffffffffffffffe301278662146975, span_id=0x2ed7e70ef543358d, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanOther', 'trulens_eval@record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac', 'trulens_eval@inputs': {'question': 'hello', 'answer': \"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\"}, 'trulens_eval@output': \"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_b9a1b716f15720549fa790baca2f08ac', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), ts=datetime.datetime(2024, 4, 26, 18, 32, 7, 458338), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11476781776, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 983465), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7769)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 889975), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7875)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 156163), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 180696)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11476641424, init_bindings=None), name='generate'))], args={'prompt': \"Relevant chunk: HELLO processed,Relevant chunk: olleh processed,Relevant chunk: I allocated 56 bytes to pretend I'm doing something. processed\"}, rets=\"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 255179), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 280144)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11484672336, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\"}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 343782), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 369505)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 431826), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458224)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), pid=71410, tid=15311948)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11484672336, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\"}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 343782), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 369505)), pid=71410, tid=15311948), tags=[], span_type='SpanOther', record_id='record_hash_b9a1b716f15720549fa790baca2f08ac', inputs={'question': 'hello', 'answer': \"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\"}, output=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None)\n", + "{'attributes': {'trulens_eval@error': None,\n", + " 'trulens_eval@inputs': {'answer': 'herp dessecorp .gnihtemos '\n", + " \"gniod m'I dneterp ot setyb \"\n", + " '65 detacolla I :knuhc '\n", + " 'tnaveleR,dessecorp hello '\n", + " ':knuhc tnaveleR,dessecorp '\n", + " 'OLLEH :knuhc tnaveleR derp '\n", + " 'and 56 bytes',\n", + " 'question': 'hello'},\n", + " 'trulens_eval@output': 'The answer to hello is probably herp '\n", + " \"dessecorp .gnihtemos gniod m'I dneterp \"\n", + " 'ot setyb 65 detacolla I :knuhc '\n", + " 'tnaveleR,dessecorp hello :knuhc '\n", + " 'tnaveleR,dessecorp OLLEH :knuhc '\n", + " 'tnaveleR derp and 56 bytes or '\n", + " 'something ...',\n", + " 'trulens_eval@record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac',\n", + " 'trulens_eval@span_type': 'SpanOther'},\n", + " 'attributes_metadata': {},\n", + " 'context': (340282366920938463461285262137769617781,\n", + " 5987148653028257066,\n", + " False,\n", + " 0,\n", + " [],\n", + " True),\n", + " 'end_timestamp': 1714156327369505000,\n", + " 'error': None,\n", + " 'events': [],\n", + " 'inputs': {'answer': \"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 \"\n", + " 'detacolla I :knuhc tnaveleR,dessecorp hello :knuhc '\n", + " 'tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 '\n", + " 'bytes',\n", + " 'question': 'hello'},\n", + " 'kind': ,\n", + " 'links': [((340282366920938463461285262137769617781,\n", + " 3375420497167857037,\n", + " False,\n", + " 0,\n", + " [],\n", + " True),\n", + " {'trulens_eval@relationship': 'parent'})],\n", + " 'name': 'fill',\n", + " 'output': 'The answer to hello is probably herp dessecorp .gnihtemos gniod '\n", + " \"m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp \"\n", + " 'hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 '\n", + " 'bytes or something ...',\n", + " 'record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac',\n", + " 'span_type': 'SpanOther',\n", + " 'start_timestamp': 1714156327343782000,\n", + " 'status': ,\n", + " 'status_description': None,\n", + " 'tags': []}\n", + "\n", + "SpanOther(name='remember', kind=, status=, status_description=None, start_timestamp=1714156327431826000, end_timestamp=1714156327458224000, context=HashableSpanContext(trace_id=0xffffffffffffffffe301278662146975, span_id=0xfc7424beeb20e002, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0xffffffffffffffffe301278662146975, span_id=0x2ed7e70ef543358d, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanOther', 'trulens_eval@record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac', 'trulens_eval@inputs': {'data': \"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\"}, 'trulens_eval@output': None, 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_b9a1b716f15720549fa790baca2f08ac', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), ts=datetime.datetime(2024, 4, 26, 18, 32, 7, 458338), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11476781776, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 983465), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7769)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 889975), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7875)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 156163), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 180696)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11476641424, init_bindings=None), name='generate'))], args={'prompt': \"Relevant chunk: HELLO processed,Relevant chunk: olleh processed,Relevant chunk: I allocated 56 bytes to pretend I'm doing something. processed\"}, rets=\"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 255179), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 280144)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11484672336, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\"}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 343782), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 369505)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 431826), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458224)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), pid=71410, tid=15311948)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 431826), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458224)), pid=71410, tid=15311948), tags=[], span_type='SpanOther', record_id='record_hash_b9a1b716f15720549fa790baca2f08ac', inputs={'data': \"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\"}, output=None, error=None)\n", + "{'attributes': {'trulens_eval@error': None,\n", + " 'trulens_eval@inputs': {'data': 'The answer to hello is '\n", + " 'probably herp dessecorp '\n", + " \".gnihtemos gniod m'I dneterp \"\n", + " 'ot setyb 65 detacolla I '\n", + " ':knuhc tnaveleR,dessecorp '\n", + " 'hello :knuhc '\n", + " 'tnaveleR,dessecorp OLLEH '\n", + " ':knuhc tnaveleR derp and 56 '\n", + " 'bytes or something ...'},\n", + " 'trulens_eval@output': None,\n", + " 'trulens_eval@record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac',\n", + " 'trulens_eval@span_type': 'SpanOther'},\n", + " 'attributes_metadata': {},\n", + " 'context': (340282366920938463461285262137769617781,\n", + " 18191205197263462402,\n", + " False,\n", + " 0,\n", + " [],\n", + " True),\n", + " 'end_timestamp': 1714156327458224000,\n", + " 'error': None,\n", + " 'events': [],\n", + " 'inputs': {'data': 'The answer to hello is probably herp dessecorp .gnihtemos '\n", + " \"gniod m'I dneterp ot setyb 65 detacolla I :knuhc \"\n", + " 'tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH '\n", + " ':knuhc tnaveleR derp and 56 bytes or something ...'},\n", + " 'kind': ,\n", + " 'links': [((340282366920938463461285262137769617781,\n", + " 3375420497167857037,\n", + " False,\n", + " 0,\n", + " [],\n", + " True),\n", + " {'trulens_eval@relationship': 'parent'})],\n", + " 'name': 'remember',\n", + " 'output': None,\n", + " 'record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac',\n", + " 'span_type': 'SpanOther',\n", + " 'start_timestamp': 1714156327431826000,\n", + " 'status': ,\n", + " 'status_description': None,\n", + " 'tags': []}\n", + "\n", + "SpanOther(name='respond_to_query', kind=, status=, status_description=None, start_timestamp=1714156326641802000, end_timestamp=1714156327458285000, context=HashableSpanContext(trace_id=0xffffffffffffffffe301278662146975, span_id=0x2ed7e70ef543358d, trace_flags=0x00, trace_state=[], is_remote=False), events=[], links={HashableSpanContext(trace_id=0xffffffffffffffffe301278662146975, span_id=0x3ca80bcf428625d5, trace_flags=0x00, trace_state=[], is_remote=False): {'trulens_eval@relationship': 'parent'}}, attributes={'trulens_eval@span_type': 'SpanOther', 'trulens_eval@record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac', 'trulens_eval@inputs': {'input': 'hello'}, 'trulens_eval@output': \"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", 'trulens_eval@error': None}, attributes_metadata={}, record=Record(record_id='record_hash_b9a1b716f15720549fa790baca2f08ac', app_id='customapp', cost=Cost(n_requests=0, n_successful_requests=0, n_classes=0, n_tokens=0, n_stream_chunks=0, n_prompt_tokens=0, n_completion_tokens=0, cost=0.0), perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), ts=datetime.datetime(2024, 4, 26, 18, 32, 7, 458338), tags='-', meta=None, main_input='hello', main_output=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", main_error=None, calls=[RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks')), RecordAppCallMethod(path=Lens().app.retriever, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_retriever.CustomRetriever, id=11476781776, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 983465), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7769)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='retrieve_chunks'))], args={'data': 'hello'}, rets=['Relevant chunk: HELLO', 'Relevant chunk: olleh', \"Relevant chunk: I allocated 56 bytes to pretend I'm doing something.\"], error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 889975), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 7875)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': 'hello'}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 156163), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 180696)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.llm, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_llm.CustomLLM, id=11476641424, init_bindings=None), name='generate'))], args={'prompt': \"Relevant chunk: HELLO processed,Relevant chunk: olleh processed,Relevant chunk: I allocated 56 bytes to pretend I'm doing something. processed\"}, rets=\"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 255179), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 280144)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.template, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomTemplate, id=11484672336, init_bindings=None), name='fill'))], args={'question': 'hello', 'answer': \"herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes\"}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 343782), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 369505)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query')), RecordAppCallMethod(path=Lens().app.memory, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_memory.CustomMemory, id=11476785296, init_bindings=None), name='remember'))], args={'data': \"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\"}, rets=None, error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 431826), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458224)), pid=71410, tid=15311948), RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), pid=71410, tid=15311948)], feedback_and_future_results=[], feedback_results=[]), call=RecordAppCall(stack=[RecordAppCallMethod(path=Lens().app, method=Method(obj=Obj(cls=examples.expositional.end2end_apps.custom_app.custom_app.CustomApp, id=11462560208, init_bindings=None), name='respond_to_query'))], args={'input': 'hello'}, rets=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None, perf=Perf(start_time=datetime.datetime(2024, 4, 26, 18, 32, 6, 641802), end_time=datetime.datetime(2024, 4, 26, 18, 32, 7, 458285)), pid=71410, tid=15311948), tags=[], span_type='SpanOther', record_id='record_hash_b9a1b716f15720549fa790baca2f08ac', inputs={'input': 'hello'}, output=\"The answer to hello is probably herp dessecorp .gnihtemos gniod m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 bytes or something ...\", error=None)\n", + "{'attributes': {'trulens_eval@error': None,\n", + " 'trulens_eval@inputs': {'input': 'hello'},\n", + " 'trulens_eval@output': 'The answer to hello is probably herp '\n", + " \"dessecorp .gnihtemos gniod m'I dneterp \"\n", + " 'ot setyb 65 detacolla I :knuhc '\n", + " 'tnaveleR,dessecorp hello :knuhc '\n", + " 'tnaveleR,dessecorp OLLEH :knuhc '\n", + " 'tnaveleR derp and 56 bytes or '\n", + " 'something ...',\n", + " 'trulens_eval@record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac',\n", + " 'trulens_eval@span_type': 'SpanOther'},\n", + " 'attributes_metadata': {},\n", + " 'context': (340282366920938463461285262137769617781,\n", + " 3375420497167857037,\n", + " False,\n", + " 0,\n", + " [],\n", + " True),\n", + " 'end_timestamp': 1714156327458285000,\n", + " 'error': None,\n", + " 'events': [],\n", + " 'inputs': {'input': 'hello'},\n", + " 'kind': ,\n", + " 'links': [((340282366920938463461285262137769617781,\n", + " 4370756423165289941,\n", + " False,\n", + " 0,\n", + " [],\n", + " True),\n", + " {'trulens_eval@relationship': 'parent'})],\n", + " 'name': 'respond_to_query',\n", + " 'output': 'The answer to hello is probably herp dessecorp .gnihtemos gniod '\n", + " \"m'I dneterp ot setyb 65 detacolla I :knuhc tnaveleR,dessecorp \"\n", + " 'hello :knuhc tnaveleR,dessecorp OLLEH :knuhc tnaveleR derp and 56 '\n", + " 'bytes or something ...',\n", + " 'record_id': 'record_hash_b9a1b716f15720549fa790baca2f08ac',\n", + " 'span_type': 'SpanOther',\n", + " 'start_timestamp': 1714156326641802000,\n", + " 'status': ,\n", + " 'status_description': None,\n", + " 'tags': []}\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/homebrew/Caskroom/miniconda/base/envs/py311_trulens/lib/python3.11/site-packages/pydantic/main.py:314: UserWarning: Pydantic serializer warnings:\n", + " Expected `Union[str, bool, int, float, json-or-python[json=list[str], python=list[str]], json-or-python[json=list[bool], python=list[bool]], json-or-python[json=list[int], python=list[int]], json-or-python[json=list[float], python=list[float]]]` but got `dict` - serialized value may not be as expected\n", + " return self.__pydantic_serializer__.to_python(\n" + ] + } + ], + "source": [ + "for span in spans:\n", + " pprint(span)\n", + " pprint(span.model_dump())\n", + " print()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "py38_trulens", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.6" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/trulens_eval/examples/expositional/end2end_apps/custom_app/custom_retriever.py b/trulens_eval/examples/expositional/end2end_apps/custom_app/custom_retriever.py index 8b65a6c57..afe49222f 100644 --- a/trulens_eval/examples/expositional/end2end_apps/custom_app/custom_retriever.py +++ b/trulens_eval/examples/expositional/end2end_apps/custom_app/custom_retriever.py @@ -1,17 +1,22 @@ import sys import time +from trulens_eval.schema import record as mod_record_schema +from trulens_eval.trace import span as mod_span from trulens_eval.tru_custom_app import instrument class CustomRetriever: + """Fake retriever.""" def __init__(self, delay: float = 0.015, alloc: int = 1024 * 1024): self.delay = delay self.alloc = alloc - # @instrument + @instrument def retrieve_chunks(self, data): + """Fake chunk retrieval.""" + temporary = [0x42] * self.alloc if self.delay > 0.0: @@ -21,3 +26,16 @@ def retrieve_chunks(self, data): f"Relevant chunk: {data.upper()}", f"Relevant chunk: {data[::-1]}", f"Relevant chunk: I allocated {sys.getsizeof(temporary)} bytes to pretend I'm doing something." ] + + @retrieve_chunks.is_span( + span_type=mod_span.SpanRetriever + ) # can also use mod_span.SpanType.RETRIEVER here + @staticmethod + def update_span( + call: mod_record_schema.RecordAppCall, + span: mod_span.SpanRetriever + ): + """Fill in span information from a recorded call to retrieve_chunks.""" + + span.input_text = call.args['data'] + span.retrieved_contexts = call.rets diff --git a/trulens_eval/trulens_eval/app.py b/trulens_eval/trulens_eval/app.py index be188b501..3ce71f4d6 100644 --- a/trulens_eval/trulens_eval/app.py +++ b/trulens_eval/trulens_eval/app.py @@ -84,6 +84,8 @@ class ComponentView(ABC): dicts representing various components, not the components themselves. """ + # TODO: replacing with trace.category.Categorizer + def __init__(self, json: JSON): self.json = json self.cls = Class.of_class_info(json) diff --git a/trulens_eval/trulens_eval/instruments.py b/trulens_eval/trulens_eval/instruments.py index db4fcac99..697ab3707 100644 --- a/trulens_eval/trulens_eval/instruments.py +++ b/trulens_eval/trulens_eval/instruments.py @@ -29,13 +29,12 @@ from trulens_eval.feedback.provider import endpoint as mod_endpoint from trulens_eval.schema import base as mod_base_schema from trulens_eval.schema import record as mod_record_schema +from trulens_eval.trace import span as mod_span from trulens_eval.utils import python from trulens_eval.utils.containers import dict_merge_with from trulens_eval.utils.imports import Dummy from trulens_eval.utils.json import jsonify -from trulens_eval.utils.pyschema import clean_attributes -from trulens_eval.utils.pyschema import Method -from trulens_eval.utils.pyschema import safe_getattr +from trulens_eval.utils import pyschema from trulens_eval.utils.python import callable_name from trulens_eval.utils.python import caller_frame from trulens_eval.utils.python import class_name @@ -208,6 +207,18 @@ def class_filter_matches(f: ClassFilter, obj: Union[Type, object]) -> bool: raise ValueError(f"Invalid filter {f}. Type, or a Tuple of Types expected.") +TSpanner = Callable[[mod_span.Span, mod_record_schema.RecordAppCall], None] + +TSpanInfo = Tuple[ + mod_span.SpanType, TSpanner +] +"""Span type and method that create spans of said type from +[RecordAppCall][trulens_eval.schema.record.RecordAppCall] objects. + +The first argument to callable is the instance of the spec specified and the +second is the [RecordAppCall][trulens_eval.schema.record.RecordAppCall] object +to fill in the span info from. +""" class Instrument(object): """Instrumentation tools.""" @@ -230,6 +241,13 @@ class Default: CLASSES = set([mod_feedback.Feedback]) """Classes to instrument.""" + SPANINFOS: Dict[pyschema.Function, TSpanInfo] = {} + """EXPERIMENTAL: Map of method to a function that + create a span from a + [RecordAppCall][trulens_eval.schema.record.RecordAppCall] made by the named + method of that class. + """ + METHODS: Dict[str, ClassFilter] = {"__call__": mod_feedback.Feedback} """Methods to instrument. @@ -266,6 +284,13 @@ def print_instrumentation(self) -> None: f = getattr(cls, method) print(f"{t*2}Method {method}: {inspect.signature(f)}") + pyfunc = pyschema.Function.of_function(f, cls=cls) + + if pyfunc in self.Default.SPANINFOS: + span_type, spanner = self.Default.SPANINFOS[pyfunc] + print(f"{t*3}Span type: {span_type}") + + print() def to_instrument_object(self, obj: object) -> bool: @@ -276,12 +301,15 @@ def to_instrument_object(self, obj: object) -> bool: # avoid issublcass checks. return any(isinstance(obj, cls) for cls in self.include_classes) - def to_instrument_class(self, cls: type) -> bool: # class + def to_instrument_class(self, cls: Union[type, pyschema.Class]) -> bool: # type=class """Determine whether the given class should be instrumented.""" # Sometimes issubclass is not supported so we return True just to be # sure we instrument that thing. + if isinstance(cls, pyschema.Class): + cls = cls.load() + try: return any( issubclass(cls, parent) for parent in self.include_classes @@ -481,7 +509,7 @@ def tru_wrapper(*args, **kwargs): stack = ctx_stacks[ctx] frame_ident = mod_record_schema.RecordAppCallMethod( - path=path, method=Method.of_method(func, obj=obj, cls=cls) + path=path, method=pyschema.Method.of_method(func, obj=obj, cls=cls) ) stack = stack + (frame_ident,) @@ -800,10 +828,10 @@ def instrument_object( # is meant to be instrumented and if so, we walk over it manually. # NOTE: some llama_index objects are using dataclasses_json but most do # not so this section applies. - attrs = clean_attributes(obj, include_props=True).keys() + attrs = pyschema.clean_attributes(obj, include_props=True).keys() if vals is None: - vals = [safe_getattr(obj, k, get_prop=True) for k in attrs] + vals = [pyschema.safe_getattr(obj, k, get_prop=True) for k in attrs] for k, v in zip(attrs, vals): @@ -903,11 +931,11 @@ def instrument_bound_methods(self, obj: object, query: Lens): if not safe_hasattr(obj, method_name): pass else: - method = safe_getattr(obj, method_name) + method = pyschema.safe_getattr(obj, method_name) print(f"\t{query}Looking at {method}") if safe_hasattr(method, "__func__"): - func = safe_getattr(method, "__func__") + func = pyschema.safe_getattr(method, "__func__") print( f"\t\t{query}: Looking at bound method {method_name} with func {func}" ) @@ -960,13 +988,41 @@ class AddInstruments(): """Utilities for adding more things to default instrumentation filters.""" @classmethod - def method(cls, of_cls: type, name: str) -> None: - """Add the class with a method named `name`, its module, and the method - `name` to the Default instrumentation walk filters.""" + def method( + cls, + of_cls: type, + name: str, + span_type: Optional[mod_span.SpanType] = None, + spanner: Optional[Callable] = None + ) -> None: + """Name a method to be instrumented. + + Args: + of_cls: The class that contains the method. + + name: The name of the method. + + span_type: The type of span to create when converting a record call + to a span. + + spanner: A callable that creates a span of the given type from a + [RecordAppCall][trulens_eval.schema.record.RecordAppCall] object. + """ Instrument.Default.MODULES.add(of_cls.__module__) Instrument.Default.CLASSES.add(of_cls) + if span_type is not None and spanner is not None: + pyfunc = pyschema.Function.of_function(getattr(of_cls, name), cls=of_cls) + + Instrument.Default.SPANINFOS[pyfunc] = (span_type, spanner) + + else: + if span_type is not None or spanner is not None: + raise ValueError( + "Both `span_type` and `spanner` must be provided if either is." + ) + check_o: ClassFilter = Instrument.Default.METHODS.get(name, ()) Instrument.Default.METHODS[name] = class_filter_disjunction( check_o, of_cls @@ -987,8 +1043,29 @@ class instrument(AddInstruments): # NOTE(piotrm): Approach taken from: # https://stackoverflow.com/questions/2366713/can-a-decorator-of-an-instance-method-access-the-class - def __init__(self, func: Callable): + def __init__( + self, + func: Optional[Callable] = None, + ): + self.func = func + self.cls = None + self.name = None + self.span_type = None + self.spanner = None + + def is_span(self, span_type: Union[mod_span.SpanType, Type[mod_span.Span]]): + """Decorator for setting span info filler function.""" + + if not isinstance(span_type, mod_span.SpanType): + span_type = mod_span.SpanType(span_type.__name__) + + self.span_type = span_type + + def set_spanner(spanner: Callable): + self.spanner = spanner + + return set_spanner def __set_name__(self, cls: type, name: str): """ @@ -998,6 +1075,19 @@ def __set_name__(self, cls: type, name: str): # Important: do this first: setattr(cls, name, self.func) + # Setup the is_span decorator to further mark the decorated method as + # one producing a span. + self.cls = cls + self.name = name + # self.func.is_span = self.is_span + # Note that this does not actually change the method, just adds it to # list of filters. - self.method(cls, name) + + self.method( + of_cls=cls, + name=name, + span_type=self.span_type, + spanner=self.spanner + ) + diff --git a/trulens_eval/trulens_eval/requirements.txt b/trulens_eval/trulens_eval/requirements.txt index 8226513bc..fd3d84275 100644 --- a/trulens_eval/trulens_eval/requirements.txt +++ b/trulens_eval/trulens_eval/requirements.txt @@ -9,6 +9,8 @@ nest-asyncio >= 1.5.8 typing_extensions >= 4.9.0 psutil >= 5.9.8 # tru.py dashboard starting/ending pip >= 24.0 # for requirements management +opentelemetry-api >= 1.24.0 # spans +opentelemetry-sdk >= 1.24.0 # spans packaging >= 23.2 # for requirements, resources management # also: resolves version conflict with langchain-core diff --git a/trulens_eval/trulens_eval/schema/record.py b/trulens_eval/trulens_eval/schema/record.py index 1d31dfeed..e1a0e378c 100644 --- a/trulens_eval/trulens_eval/schema/record.py +++ b/trulens_eval/trulens_eval/schema/record.py @@ -32,6 +32,9 @@ class RecordAppCallMethod(serial.SerialModel): method: pyschema.Method """The method that was called.""" + def __hash__(self): + return hash((self.path. self.method)) + class RecordAppCall(serial.SerialModel): """Info regarding each instrumented method call.""" @@ -65,12 +68,19 @@ def top(self) -> RecordAppCallMethod: return self.stack[-1] + def caller(self) -> Optional[RecordAppCallMethod]: + """The frame right under the top of the stack, i.e. the caller of this frame.""" + + if len(self.stack) < 2: + return None + + return self.stack[-2] + def method(self) -> pyschema.Method: """The method at the top of the stack.""" return self.top().method - class Record(serial.SerialModel, Hashable): """The record of a single main method call. @@ -207,7 +217,6 @@ def layout_calls_as_app(self) -> Bunch: return ret - # HACK013: Need these if using __future__.annotations . RecordAppCallMethod.model_rebuild() Record.model_rebuild() diff --git a/trulens_eval/trulens_eval/trace/__init__.py b/trulens_eval/trulens_eval/trace/__init__.py new file mode 100644 index 000000000..f578bab8d --- /dev/null +++ b/trulens_eval/trulens_eval/trace/__init__.py @@ -0,0 +1,263 @@ +"""Tracing and spans. + +This module contains a [OTSpan][trulens_eval.trace.OTSpan], a Span +implementation conforming to the OpenTelemetry span specification and related +utilities. These are further specialized in +[Span][trulens_eval.trace.span.Span]. +""" + +from __future__ import annotations + +from logging import getLogger +import time +from typing import (ClassVar, Dict, List, Mapping, Optional, Tuple, TypeVar, + Union) + +from opentelemetry.trace import status as trace_status +import opentelemetry.trace as ot_trace +import opentelemetry.trace.span as ot_span +from opentelemetry.util import types as ot_types +import pydantic +from pydantic import PlainSerializer +from pydantic.functional_validators import PlainValidator +from typing_extensions import Annotated + +logger = getLogger(__name__) + +# Type alises + +A = TypeVar("A") +B = TypeVar("B") + +TTimestamp = int +"""Type of timestamps in spans. + +64 bit int representing nanoseconds since epoch as per OpenTelemetry. +""" +NUM_TIMESTAMP_BITS = 64 + +TSpanID = int # +"""Type of span identifiers. + +64 bit int as per OpenTelemetry. +""" +NUM_SPANID_BITS = 64 +"""Number of bits in a span identifier.""" + +TTraceID = int +"""Type of trace identifiers. + +128 bit int as per OpenTelemetry. +""" +NUM_TRACEID_BITS = 128 +"""Number of bits in a trace identifier.""" + +T = TypeVar("T") + +class HashableSpanContext(ot_span.SpanContext): + """SpanContext that can be hashed. + + Does not change data layout or behaviour. Changing SpanContext + `__class__` with this should be safe. + """ + + def __hash__(self): + return hash((self.trace_id, self.span_id)) + + def __eq__(self, other): + return self.trace_id == other.trace_id and self.span_id == other.span_id + +def deserialize_contextmapping( + v: List[Tuple[HashableSpanContext, T]] +) -> Dict[HashableSpanContext, T]: + """Deserialize a list of tuples as a dictionary.""" + + # skip last element of SpanContext as it is computed in SpanContext.__init__ from others + return {HashableSpanContext(*k[0:5]): v for k, v in v} + +def serialize_contextmapping( + v: Dict[HashableSpanContext, T], +) -> List[Tuple[HashableSpanContext, T]]: + """Serialize a dictionary as a list of tuples.""" + + return list(v.items()) + +ContextMapping = Annotated[ + Dict[HashableSpanContext, T], + PlainSerializer(serialize_contextmapping), + PlainValidator(deserialize_contextmapping) +] +"""Type annotation for pydantic fields that store dictionaries whose keys are +HashableSpanContext. + +This is needed to help pydantic figure out how to serialize and deserialize these dicts. +""" + + +def make_hashable(context: ot_span.SpanContext) -> HashableSpanContext: + # HACK015: replace class of contexts to add hashing + + if context.__class__ is not HashableSpanContext: + context.__class__ = HashableSpanContext + + # Return not needed but useful for type checker. + return context + + +class OTSpan(pydantic.BaseModel, ot_span.Span): + """Implementation of OpenTelemetry Span requirements. + + See also [OpenTelemetry Span](https://opentelemetry.io/docs/specs/otel/trace/api/#span). + """ + + _vendor: ClassVar[str] = "trulens_eval" + """Vendor name as per OpenTelemetry attribute keys specifications.""" + + @classmethod + def vendor_attr(cls, name: str) -> str: + """Add vendor prefix to attribute name.""" + + return f"{cls._vendor}@{name}" + + model_config = { + 'arbitrary_types_allowed': True, + 'use_attribute_docstrings': True + } + """Pydantic configuration.""" + + name: str + """Name of span.""" + + kind: ot_trace.SpanKind = ot_trace.SpanKind.INTERNAL + """Kind of span.""" + + status: trace_status.StatusCode = trace_status.StatusCode.UNSET + """Status of the span as per OpenTelemetry Span requirements.""" + + status_description: Optional[str] = None + """Status description as per OpenTelemetry Span requirements.""" + + start_timestamp: TTimestamp = pydantic.Field(default_factory=time.time_ns) + """Timestamp when the span's activity started in nanoseconds since epoch.""" + + end_timestamp: Optional[TTimestamp] = None + """Timestamp when the span's activity ended in nanoseconds since epoch. + + None if not yet ended. + """ + + context: HashableSpanContext + """Unique immutable identifier for the span.""" + + events: List[Tuple[str, ot_types.Attributes, TTimestamp]] = \ + pydantic.Field(default_factory=list) + """Events recorded in the span.""" + + links: ContextMapping[Mapping[str, ot_types.AttributeValue]] = \ + pydantic.Field(default_factory=dict) + """Relationships to other spans with attributes on each link.""" + + attributes: Dict[str, ot_types.AttributeValue] = \ + pydantic.Field(default_factory=dict) + """Attributes of span.""" + + def __init__(self, name: str, context: ot_span.SpanContext, **kwargs): + kwargs['name'] = name + kwargs['context'] = make_hashable(context) + kwargs['attributes'] = kwargs.get('attributes', {}) or {} + kwargs['links'] = kwargs.get('links', {}) or {} + + super().__init__(**kwargs) + + def end(self, end_time: Optional[TTimestamp] = None): + """See [end][opentelemetry.trace.span.Span.end]""" + + if end_time: + self.end_timestamp = end_time + else: + self.end_timestamp = time.time_ns() + + self.status = trace_status.StatusCode.OK + + def get_span_context(self) -> ot_span.SpanContext: + """See [get_span_context][opentelemetry.trace.span.Span.get_span_context].""" + + return self.context + + def set_attributes(self, attributes: Dict[str, ot_types.AttributeValue]) -> None: + """See [set_attributes][opentelemetry.trace.span.Span.set_attributes].""" + + self.attributes.update(attributes) + + def set_attribute(self, key: str, value: ot_types.AttributeValue) -> None: + """See [set_attribute][opentelemetry.trace.span.Span.set_attribute].""" + + self.attributes[key] = value + + def add_event( + self, + name: str, + attributes: ot_types.Attributes = None, + timestamp: Optional[TTimestamp] = None + ) -> None: + """See [add_event][opentelemetry.trace.span.Span.add_event].""" + + self.events.append((name, attributes, timestamp or time.time_ns())) + + def add_link( + self, + context: ot_span.SpanContext, + attributes: ot_types.Attributes = None + ) -> None: + """See [add_link][opentelemetry.trace.span.Span.add_link].""" + + context = make_hashable(context) + + if attributes is None: + attributes = {} + + self.links[context] = attributes + + def update_name(self, name: str) -> None: + """See [update_name][opentelemetry.trace.span.Span.update_name].""" + + self.name = name + + def is_recording(self) -> bool: + """See [is_recording][opentelemetry.trace.span.Span.is_recording].""" + + return self.status == trace_status.StatusCode.UNSET + + def set_status( + self, + status: Union[ot_span.Status, ot_span.StatusCode], + description: Optional[str] = None + ) -> None: + """See [set_status][opentelemetry.trace.span.Span.set_status].""" + + if isinstance(status, ot_span.Status): + if description is not None: + raise ValueError("Ambiguous status description provided both in `status.description` and in `description`.") + + self.status = status.status_code + self.status_description = status.description + else: + self.status = status + self.status_description = description + + def record_exception( + self, + exception: Exception, + attributes: ot_types.Attributes = None, + timestamp: Optional[TTimestamp] = None, + escaped: bool = False + ) -> None: + """See [record_exception][opentelemetry.trace.span.Span.record_exception].""" + + self.status = trace_status.StatusCode.ERROR + + self.add_event( + self.vendor_attr("exception"), + attributes, + timestamp + ) diff --git a/trulens_eval/trulens_eval/trace/category.py b/trulens_eval/trulens_eval/trace/category.py new file mode 100644 index 000000000..9e7152311 --- /dev/null +++ b/trulens_eval/trulens_eval/trace/category.py @@ -0,0 +1,345 @@ +"""Span categorization and category detection.""" + +from __future__ import annotations + +import logging +from typing import List, Optional, Sequence, Set, TypeVar + +from trulens_eval import instruments as mod_instruments +from trulens_eval import trace as mod_trace +from trulens_eval.utils import containers as mod_containers_utils +from trulens_eval.trace import tracer as mod_tracer +from trulens_eval.schema import record as mod_record_schema +from trulens_eval.trace import span as mod_span +from trulens_eval.trace import tracer as mod_tracer +from trulens_eval.utils import pyschema + +T = TypeVar("T") + +logger = logging.getLogger(__name__) + +class Categorizer(): + """Categorizes RecordAppCalls into Spans of various types.""" + + known_modules = set([ + "langchain", + "llama_index", + "nemoguardrails", + "trulens_eval" + ]) + + @staticmethod + def class_is(pycls: pyschema.Class) -> bool: + """ + Determine whether the given class representation `pycls` is of the type to + be viewed as this component type. + """ + + return True + + @staticmethod + def innermost_base( + bases: Optional[Sequence[pyschema.Class]] = None, + among_modules: Optional[Set[str]] = None + ) -> Optional[str]: + """ + Given a sequence of classes, return the first one which comes from one + of the `among_modules`. You can use this to determine where ultimately + the encoded class comes from in terms of langchain, llama_index, or + trulens_eval even in cases they extend each other's classes. Returns + None if no module from `among_modules` is named in `bases`. + """ + if among_modules is None: + among_modules = Categorizer.known_modules + + if bases is None: + return None + + for base in bases: + if "." in base.module.module_name: + root_module = base.module.module_name.split(".")[0] + else: + root_module = base.module.module_name + + if root_module in among_modules: + return root_module + + return None + + @staticmethod + def span_of_call( + call: mod_record_schema.RecordAppCall, + tracer: mod_tracer.Tracer, + context: Optional[mod_trace.HashableSpanContext] = None + ) -> mod_span.Span: + """Categorizes a [RecordAppCall][trulens_eval.schema.record.RecordAppCall] into a span. + + Args: + call: The call to categorize. + + tracer: The tracer to create the span in. + + context: The context of the parent span if any. + + Returns: + + The span. + """ + + method = call.method() + package_name = method.obj.cls.module.package_name + + subcategorizer = None + subs = [ + LangChainCategorizer, + LlamaIndexCategory, + NemoGuardRailsCategorizer, + TruLensCategorizer, + CustomCategorizer + ] + + if package_name is None: + logger.warning("Unknown package.") + + for sub in subs: + if sub.class_is(method.obj.cls): + subcategorizer = sub + break + + if subcategorizer is not None: + return subcategorizer.span_of_call( + call=call, tracer=tracer, context=context + ) + + span = tracer.new_span( + name = method.name, + cls = mod_span.SpanOther, # if no category known + context = context + ) + + return span + + @staticmethod + def spans_of_record(record: mod_record_schema.Record) -> List[mod_span.Span]: + """Convert this record into a tracer with all of the calls populated as spans.""" + + # Init with trace_id that is determined by record_id. + tracer = mod_trace.tracer.Tracer( + trace_id=mod_tracer.trace_id_of_string_id(record.record_id) + ) + + root_span = tracer.new_span( + name="root", + cls=mod_span.SpanRoot, + start_time=mod_containers_utils.ns_timestamp_of_datetime(record.perf.start_time) + ) + root_span.end(mod_containers_utils.ns_timestamp_of_datetime(record.perf.end_time)) + + # TransSpanRecord fields + root_span.record = record + root_span.record_id = record.record_id + + method_to_span_map = {} + + for call in record.calls: + method = call.top() + + span = Categorizer.span_of_call( + call=call, + tracer=tracer, + context=root_span.context # might be changed below + ) + + method_to_span_map[method] = span + + # OTSpan fields + span.start_timestamp = mod_containers_utils.ns_timestamp_of_datetime(call.perf.start_time) + span.end(mod_containers_utils.ns_timestamp_of_datetime(call.perf.end_time)) + + # TransSpanRecord fields + span.record_id = record.record_id + span.record = record + + # TransSpanRecordAppCall fields + span.call = call + span.inputs = call.args + span.output = call.rets + span.error = call.error + + # Add to traces. Not needed now but might be later. + tracer.spans[span.context] = span + + # Update parent context links. + for span in tracer.spans.values(): + if isinstance(span, mod_span.TransSpanRecordAppCall): + + parent_method = span.call.caller() + if parent_method in method_to_span_map: + + parent_span = method_to_span_map[parent_method] + span.parent_context = parent_span.context + + return list(tracer.spans.values()) + + +class LangChainCategorizer(Categorizer): + """Categorizer for _LangChain_ classes.""" + + @staticmethod + def class_is(pycls: pyschema.Class) -> bool: + if Categorizer.innermost_base(pycls.bases) == "langchain": + return True + + return False + + @staticmethod + def span_of_call( + call: mod_record_schema.RecordAppCall, + tracer: mod_tracer.Tracer, + context: Optional[mod_trace.HashableSpanContext] = None + ) -> mod_span.Span: + """Converts a call by a _LangChain_ class into the appropriate span.""" + + pycls = call.method().obj.cls + + if pycls.noserio_issubclass( + module_name="langchain.prompts.base", + class_name="BasePromptTemplate" + ) or pycls.noserio_issubclass( + module_name="langchain.schema.prompt_template", + class_name="BasePromptTemplate" + ): # langchain >= 0.230 + # Prompt + pass + + elif pycls.noserio_issubclass( + module_name="langchain.llms.base", class_name="BaseLLM" + ): + # LLM + span = tracer.new_span( + name = call.method().name, + cls = mod_span.SpanLLM, + context = context + ) + span.model_name = "TBD" + + return tracer.new_span( + name = call.method().name, + cls = mod_span.SpanOther, + context = context + ) + +class LlamaIndexCategory(Categorizer): + """Categorizer for _LlamaIndex_ classes.""" + + @staticmethod + def class_is(pycls: pyschema.Class) -> bool: + if Categorizer.innermost_base(pycls.bases) == "llama_index": + return True + + return False + + @staticmethod + def span_of_call( + call: mod_record_schema.RecordAppCall, + tracer: mod_tracer.Tracer, + context: Optional[mod_trace.HashableSpanContext] = None + ) -> mod_span.Span: + """Converts a call by a _LlamaIndex_ class into the appropriate span.""" + + pycls = call.method().obj.cls + + if pycls.noserio_issubclass( + module_name="llama_index.prompts.base", class_name="Prompt" + ): + # Prompt + pass + + elif pycls.noserio_issubclass( + module_name="llama_index.agent.types", class_name="BaseAgent" + ): + # Agent + pass + + elif pycls.noserio_issubclass( + module_name="llama_index.tools.types", class_name="BaseTool" + ): + # Tool + pass + + elif pycls.noserio_issubclass( + module_name="llama_index.llms.base", class_name="LLM" + ): + # LLM + span = tracer.new_span( + name = call.method().name, + cls = mod_span.SpanLLM, + context = context + ) + span.model_name = "TBD" + + return tracer.new_span( + name = call.method().name, + cls = mod_span.SpanOther, + context = context + ) + + +class NemoGuardRailsCategorizer(Categorizer): + """Categorizer for _Nemo GuardRails_ classes.""" + + @staticmethod + def class_is(pycls: pyschema.Class) -> bool: + if Categorizer.innermost_base(pycls.bases) == "nemoguardrails": + return True + + return False + +class TruLensCategorizer(Categorizer): + """Categorizer for _TruLens-Eval_ classes.""" + + @staticmethod + def class_is(pycls: pyschema.Class) -> bool: + if Categorizer.innermost_base(pycls.bases) == "trulens_eval": + return True + + return False + +class CustomCategorizer(Categorizer): + """Categorizer for custom classes annotated with + [instrument][trulens_eval.tru_custom_app.instrument] or related.""" + + @staticmethod + def class_is(pycls: pyschema.Class) -> bool: + i = mod_instruments.Instrument() + return i.to_instrument_class(pycls) + + @staticmethod + def span_of_call( + call: mod_record_schema.RecordAppCall, + tracer: mod_tracer.Tracer, + context: Optional[mod_trace.HashableSpanContext] = None + ) -> mod_span.Span: + """Converts a custom instrumentation-annotated method call into the appropriate span.""" + + pymethod = call.method() + pyfunc = pymethod.as_func() + method_name = pymethod.name + + span_info = mod_instruments.Instrument.Default.SPANINFOS.get(pyfunc) + + if span_info is not None: + cls = span_info[0].to_class() + else: + cls = mod_span.SpanOther + + span = tracer.new_span( + name = method_name, + context = context, + cls=cls + ) + + if span_info is not None: + span_info[1](call=call, span=span) + + return span diff --git a/trulens_eval/trulens_eval/trace/span.py b/trulens_eval/trulens_eval/trace/span.py new file mode 100644 index 000000000..d88796026 --- /dev/null +++ b/trulens_eval/trulens_eval/trace/span.py @@ -0,0 +1,356 @@ +"""Spans + +These are roughly equivalent to `RecordAppCall` but abstract away specific method +information into type of call related to types of components. +""" + +from __future__ import annotations + +import datetime +from enum import Enum +from logging import getLogger +from typing import Any, Callable, Dict, List, Optional, Type, TypeVar + +from opentelemetry.util import types as ot_types +import pandas as pd +from pydantic import computed_field +from pydantic import Field +from pydantic import TypeAdapter + +from trulens_eval import trace as mod_trace +from trulens_eval.schema import record as mod_record_schema +from trulens_eval.utils import containers as mod_container_utils + +logger = getLogger(__name__) + +T = TypeVar("T") + +class Span(mod_trace.OTSpan): + """Base Span type. + + Smallest unit of recorded activity. + """ + + @staticmethod + def attribute_property( + name: str, + typ: Optional[Type[T]] = None, + typ_factory: Optional[Callable[[], Type[T]]] = None, + default: Optional[T] = None, + default_factory: Optional[Callable[[], T]] = None + ) -> property: + """Utility for creating properties that stores their values in the + attributes dictionary with a vendor prefix. + + Validates default and on assignment. + + Args: + name: The name of the property. The key used for storage will be + this with the vendor prefix. + + typ: The type of the property. + + typ_factory: A factory function that returns the type of the + property. This can be used for forward referenced types. + + default: The default value of the property. + + default_factory: A factory function that returns the default value + of the property. This can be used for defaults that make use of + forward referenced types. + """ + initialized = False + tadapter = None + + def initialize(): + # Delaying the steps in this method until the first time the + # property is used as otherwise forward references might not be + # ready. + + nonlocal initialized, tadapter + + if initialized: + return + + nonlocal typ, default + if typ is None and typ_factory is not None: + typ = typ_factory() + + if default is None and default_factory is not None: + default = default_factory() + + if typ is None and default is not None: + typ = type(default) + + if typ is None: + tadapter = None + else: + tadapter = TypeAdapter(typ) + if default is not None: + tadapter.validate_python(default) + + def getter(self) -> T: + initialize() + return self.attributes.get(self.vendor_attr(name), default) + + def setter(self, value: T) -> None: + initialize() + if tadapter is not None: + tadapter.validate_python(value) + + self.attributes[self.vendor_attr(name)] = value + + prop = property(getter, setter) + + return computed_field(prop) + + @property + def start_datetime(self) -> datetime.datetime: + """Start time of span as a [datetime][datetime.datetime].""" + return mod_container_utils.datetime_of_ns_timestamp(self.start_timestamp) + + @start_datetime.setter + def start_datetime(self, value: datetime.datetime): + self.start_timestamp = mod_container_utils.ns_timestamp_of_datetime(value) + + @property + def end_datetime(self) -> datetime.datetime: + """End time of span as a [datetime][datetime.datetime].""" + return mod_container_utils.datetime_of_ns_timestamp(self.end_timestamp) + + @end_datetime.setter + def end_datetime(self, value: datetime.datetime): + self.end_timestamp = mod_container_utils.ns_timestamp_of_datetime(value) + + @property + def span_id(self) -> mod_trace.TSpanID: + """Identifier for the span.""" + + return self.context.span_id + + @property + def trace_id(self) -> mod_trace.TTraceID: + """Identifier for the trace this span belongs to.""" + + return self.context.trace_id + + @property # want # @functools.cached_property but those are not allowed to have setters + def parent_context(self) -> Optional[mod_trace.HashableSpanContext]: + """Context of parent span if any. + + This is stored in OT links with a relationship attribute of "parent". + None if this is a root span or otherwise it does not have a parent. + """ + + for link_context, link_attributes in self.links.items(): + if link_attributes.get(self.vendor_attr("relationship")) == "parent": + return link_context + + return None + + @parent_context.setter + def parent_context(self, value: Optional[mod_trace.HashableSpanContext]): + if value is None: + return + + if self.parent_context is not None: + # Delete existing parent if any. + del self.links[self.parent_context] + + self.add_link(value, {self.vendor_attr("relationship"): "parent"}) + + # want functools.cached_property but need updating due to the above setter + @property + def parent_span_id(self) -> Optional[mod_trace.TSpanID]: + """Id of parent span if any.""" + + parent_context = self.parent_context + if parent_context is not None: + return parent_context.span_id + + return None + + tags = attribute_property( + "tags", typ=List[str], default_factory=list + ) + """Tags associated with the span.""" + + span_type = attribute_property( + "span_type", + typ_factory=lambda: SpanType, + default_factory=lambda: SpanType.UNTYPED + ) + """Type of span.""" + + attributes_metadata: mod_container_utils.DictNamespace[ot_types.AttributeValue] + # will be set as a DictNamespace indexing elements in attributes + @property + def metadata(self) -> mod_container_utils.DictNamespace[ot_types.AttributeValue]: + return self.attributes_metadata + + @metadata.setter + def metadata(self, value: Dict[str, str]): + for k, v in value.items(): + self.attributes_metadata[k] = v + + def __init__(self, **kwargs): + kwargs['attributes_metadata'] = mod_container_utils.DictNamespace(parent={}, namespace="temp") + # Temporary fake for validation in super.__init__ below. + + super().__init__(**kwargs) + + # Actual. This is needed as pydantic will copy attributes dict in init. + self.attributes_metadata = mod_container_utils.DictNamespace( + parent=self.attributes, + namespace=self.vendor_attr("metadata") + ) + + self.set_attribute(self.vendor_attr("span_type"), self.__class__.__name__) + +class SpanUntyped(Span): + """Generic span type. + + This represents spans that are being recorded but have not yet been + determined to be of a particular type. + """ + +class TransSpanRecord(Span): + """A span whose activity was recorded in a record. + + Features references to the record. + + !!! note + This is a transitional type for the traces work. + """ + + record: mod_record_schema.Record = Field(exclude=True, default=None) + record_id = Span.attribute_property("record_id", typ=str, default=None) + +class SpanMethodCall(TransSpanRecord): + """Span which corresponds to a method call. + + See also temporary development attributes in + [TransSpanRecordAppCall][trulens_eval.trace.span.TransSpanRecordCall]. + """ + + inputs = Span.attribute_property("inputs", typ=Optional[Dict[str, Any]], default_factory=None) + # TODO: Need to encode to OT AttributeValue + + output = Span.attribute_property("output", typ=Optional[Any], default_factory=None) + # TODO: Need to encode to OT AttributeValue + + error = Span.attribute_property("error", typ=Optional[Any], default_factory=None) + # TODO: Need to encode to OT AttributeValue + + +class TransSpanRecordAppCall(SpanMethodCall): + """A Span which corresponds to single + [RecordAppCall][trulens_eval.schema.record.RecordAppCall]. + + Features references to the call. + + !!! note + This is a transitional type for the traces work. The non-transitional + fields are being placed in + [SpanMethodCall][trulens_eval.trace.span.SpanMethodCall] instead. + """ + call: mod_record_schema.RecordAppCall = Field(exclude=True, default=None) + + +class SpanRoot(TransSpanRecord): + """A root span encompassing some collection of spans. + + Does not indicate any particular activity by itself beyond its children. + """ + +SpanTyped = TransSpanRecordAppCall +"""Alias for the superclass of spans that went through the record call conversion.""" + + +class SpanRetriever(SpanTyped): + """A retrieval.""" + + input_text = Span.attribute_property("input_text", str) + """Input text whose related contexts are being retrieved.""" + + input_embedding = Span.attribute_property("input_embedding", list)#List[float]) + """Embedding of the input text.""" + + distance_type = Span.attribute_property("distance_type", str) + """Distance function used for ranking contexts.""" + + num_contexts = Span.attribute_property("num_contexts", int) + """The number of contexts requested, not necessarily retrieved.""" + + retrieved_contexts = Span.attribute_property("retrieved_contexts", list)#List[str]) + """The retrieved contexts.""" + +class SpanReranker(SpanTyped): + """A reranker call.""" + +class SpanLLM(SpanTyped): + """A generation call to an LLM.""" + + model_name = Span.attribute_property("model_name", str) + """The model name of the LLM.""" + +class SpanEmbedding(SpanTyped): + """An embedding cal.""" + +class SpanTool(SpanTyped): + """A tool invocation.""" + +class SpanAgent(SpanTyped): + """An agent invocation.""" + +class SpanTask(SpanTyped): + """A task invocation.""" + +class SpanOther(SpanTyped): + """Other uncategorized spans.""" + +class SpanType(Enum): + """Span types. + + This is a bit redundant with the span type hierarchy above. It is here for + convenience of looking up types in means other than `__class__` or via + `isinstance`. + """ + + def to_class(self) -> Type[Span]: + """Convert to the class for this type.""" + + if hasattr(mod_trace.span, self.value): + return getattr(mod_trace.span, self.value) + + raise ValueError(f"Span type {self.value} not found in module.") + + UNTYPED = SpanUntyped.__name__ + """See [SpanUntyped][trulens_eval.trace.span.SpanUntyped].""" + + ROOT = SpanRoot.__name__ + """See [SpanRoot][trulens_eval.trace.span.SpanRoot].""" + + RETRIEVER = SpanRetriever.__name__ + """See [SpanRetriever][trulens_eval.trace.span.SpanRetriever].""" + + RERANKER = SpanReranker.__name__ + """See [SpanReranker][trulens_eval.trace.span.SpanReranker].""" + + LLM = SpanLLM.__name__ + """See [SpanLLM][trulens_eval.trace.span.SpanLLM].""" + + EMBEDDING = SpanEmbedding.__name__ + """See [SpanEmbedding][trulens_eval.trace.span.SpanEmbedding].""" + + TOOL = SpanTool.__name__ + """See [SpanTool][trulens_eval.trace.span.SpanTool].""" + + AGENT = SpanAgent.__name__ + """See [SpanAgent][trulens_eval.trace.span.SpanAgent].""" + + TASK = SpanTask.__name__ + """See [SpanTask][trulens_eval.trace.span.SpanTask].""" + + OTHER = SpanOther.__name__ + """See [SpanOther][trulens_eval.trace.span.SpanOther].""" diff --git a/trulens_eval/trulens_eval/trace/tracer.py b/trulens_eval/trulens_eval/trace/tracer.py new file mode 100644 index 000000000..fb20575e8 --- /dev/null +++ b/trulens_eval/trulens_eval/trace/tracer.py @@ -0,0 +1,204 @@ +"""Tracer, manages spans.""" + +from __future__ import annotations + +import contextvars +from logging import getLogger +import random +from typing import Iterator, Mapping, Optional, Type + +import opentelemetry +import opentelemetry.trace as ot_trace +import opentelemetry.trace.span as ot_span +from opentelemetry.util import types as ot_types +from opentelemetry.util._decorator import _agnosticcontextmanager +import pydantic + +from trulens_eval import trace as mod_trace +from trulens_eval.trace import span as mod_span + +logger = getLogger(__name__) + +def trace_id_of_string_id(s: str) -> mod_trace.TTraceID: + """Convert a string id to a trace ID. + + Not an OT requirement. + """ + + return hash(s) % (1 << mod_trace.NUM_TRACEID_BITS) + +def span_id_of_string_id(s: str) -> mod_trace.TSpanID: + """Convert a string id to a span ID. + + Not an OT requirement. + """ + + return hash(s) % (1 << mod_trace.NUM_SPANID_BITS) + +class Tracer(pydantic.BaseModel, ot_trace.Tracer): + """Implementation of OpenTelemetry Tracer requirements.""" + + model_config = { + 'arbitrary_types_allowed': True, + 'use_attribute_docstrings': True + } + """Pydantic configuration.""" + + stack: contextvars.ContextVar[mod_trace.HashableSpanContext] = pydantic.Field( + default_factory=lambda: contextvars.ContextVar("stack", default=None), + exclude=True + ) + + instrumenting_module_name: str = "trulens_eval" + instrumenting_library_version: Optional[str] = None#trulens_eval.__version__ + + spans: mod_trace.ContextMapping[ + Mapping[str, ot_types.AttributeValue], + ] = pydantic.Field(default_factory=dict) + """Spans recorded by the tracer.""" + + state: ot_span.TraceState = pydantic.Field(default_factory=ot_span.TraceState) + """Trace attributes.""" + + trace_id: mod_trace.TTraceID + """Unique identifier for the trace.""" + + def __init__( + self, + trace_id: Optional[mod_trace.TTraceId] = None, + **kwargs + ): + if trace_id is None: + trace_id = random.getrandbits(mod_trace.NUM_TRACEID_BITS) + + kwargs['trace_id'] = trace_id + + super().__init__(**kwargs) + + def new_span( + self, + name: str, + cls: Type[mod_span.Span], + context: Optional[ot_trace.Context] = None, + kind: ot_trace.SpanKind = ot_trace.SpanKind.INTERNAL, + attributes: ot_trace.types.Attributes = None, + links: ot_trace._Links = None, + start_time: Optional[int] = None, + record_exception: bool = True, + set_status_on_exception: bool = True + ) -> mod_trace.Span: + """See [new_span][opentelemetry.trace.Tracer.new_span].""" + + span_context = mod_trace.HashableSpanContext( + trace_id=self.trace_id, + span_id=random.getrandbits(mod_trace.NUM_SPANID_BITS), + is_remote=False, + trace_state = self.state + ) + + span = cls( + name=name, + context=span_context, + kind=kind, + attributes=attributes, + links=links, + start_time=start_time, + record_exception=record_exception, + set_status_on_exception=set_status_on_exception + ) + + if context is not None: + context = mod_trace.make_hashable(context) + span.add_link(context, {mod_span.Span.vendor_attr("relationship"): "parent"}) + + self.spans[span_context] = span + + return span + + def start_span( + self, + name: str, + context: Optional[ot_trace.Context] = None, + kind: ot_trace.SpanKind = ot_trace.SpanKind.INTERNAL, + attributes: ot_trace.types.Attributes = None, + links: ot_trace._Links = None, + start_time: Optional[int] = None, + record_exception: bool = True, + set_status_on_exception: bool = True, + ) -> mod_span.Span: + """See [start_span][opentelemetry.trace.Tracer.start_span].""" + + if context is None: + parent_context = self.stack.get() + + else: + parent_context = mod_trace.make_hashable(context) + + if parent_context.trace_id != self.trace_id: + logger.warning("Parent context is not being traced by this tracer.") + + span_context = mod_trace.HashableSpanContext( + trace_id=self.trace_id, + span_id=random.getrandbits(mod_trace.NUM_SPANID_BITS), + is_remote=False, + trace_state = self.state # unsure whether these should be shared across all spans produced by this tracer + ) + + span = mod_span.SpanUntyped( + name=name, + context=span_context, + kind=kind, + attributes=attributes, + links=links, + start_time=start_time, + record_exception=record_exception, + set_status_on_exception=set_status_on_exception + ) + + if parent_context is not None: + span.add_link(parent_context, {mod_span.Span.vendor_attr("relationship"): "parent"}) + + self.spans[span_context] = span + + return span + + @_agnosticcontextmanager + def start_as_current_span( + self, + name: str, + context: Optional[ot_trace.Context] = None, + kind: ot_trace.SpanKind = opentelemetry.trace.SpanKind.INTERNAL, + attributes: ot_types.Attributes = None, + links: ot_trace._Links = None, + start_time: Optional[int] = None, + record_exception: bool = True, + set_status_on_exception: bool = True, + end_on_exit: bool = True, + ) -> Iterator[mod_span.Span]: + """See [start_as_current_span][opentelemetry.trace.Tracer.start_as_current_span].""" + + if context is not None: + context = mod_trace.make_hashable(context) + + span = self.start_span( + name, + context, + kind, + attributes, + links, + start_time, + record_exception, + set_status_on_exception + ) + + token = self.stack.set(span.context) + + # Unsure if this ot_trace stuff is needed. + span_token = ot_trace.use_span(span, end_on_exit=end_on_exit).__enter__() + yield span + + # Same + span_token.__exit__(None, None, None) + + self.stack.reset(token) + return diff --git a/trulens_eval/trulens_eval/tru_custom_app.py b/trulens_eval/trulens_eval/tru_custom_app.py index 6a2ee9770..89f1f9264 100644 --- a/trulens_eval/trulens_eval/tru_custom_app.py +++ b/trulens_eval/trulens_eval/tru_custom_app.py @@ -199,6 +199,7 @@ class will not be found by trulens. from trulens_eval import app as mod_app from trulens_eval.instruments import Instrument from trulens_eval.instruments import instrument as base_instrument +from trulens_eval.trace import span as mod_span from trulens_eval.utils.pyschema import Class from trulens_eval.utils.pyschema import Function from trulens_eval.utils.pyschema import FunctionOrMethod @@ -531,12 +532,20 @@ class instrument(base_instrument): """ @classmethod - def method(self_class, cls: type, name: str) -> None: - base_instrument.method(cls, name) + def method( + cls, + of_cls: type, + name: str, + span_type: Optional[mod_span.SpanType] = None, + spanner: Optional[Callable] = None + ) -> None: + base_instrument.method( + of_cls=of_cls, name=name, span_type=span_type, spanner=spanner + ) # Also make note of it for verification that it was found by the walk # after init. - TruCustomApp.functions_to_instrument.add(getattr(cls, name)) + TruCustomApp.functions_to_instrument.add(getattr(of_cls, name)) import trulens_eval # for App class annotations diff --git a/trulens_eval/trulens_eval/utils/containers.py b/trulens_eval/trulens_eval/utils/containers.py index 9dd83f8a9..3eddb92f8 100644 --- a/trulens_eval/trulens_eval/utils/containers.py +++ b/trulens_eval/trulens_eval/utils/containers.py @@ -4,11 +4,14 @@ from __future__ import annotations +import datetime import itertools import logging from pprint import PrettyPrinter from typing import Callable, Dict, Iterable, Sequence, Tuple, TypeVar, Union +import pandas as pd + logger = logging.getLogger(__name__) pp = PrettyPrinter() @@ -16,6 +19,39 @@ A = TypeVar("A") B = TypeVar("B") +# Time container utilities + +def datetime_of_ns_timestamp(timestamp: int) -> datetime.datetime: + """Convert a nanosecond timestamp to a datetime.""" + return pd.Timestamp(timestamp, unit='ns').to_pydatetime() + +def ns_timestamp_of_datetime(dt: datetime.datetime) -> int: + """Convert a datetime to a nanosecond timestamp.""" + return pd.Timestamp(dt).as_unit('ns').value + +# Dicts utilities + +class DictNamespace(Dict[str, T]): + """View into a dict with keys prefixed by some `namespace` string. + + Replicates the values without the prefix in self. + """ + + def __init__(self, parent: Dict[str, T], namespace: str, **kwargs): + self.parent = parent + self.namespace = namespace + + def __getitem__(self, key: str) -> T: + return dict.__getitem__(self, key) + + def __setitem__(self, key: str, value: T) -> None: + dict.__setitem__(self, key, value) + self.parent[f"{self.namespace}.{key}"] = value + + def __delitem__(self, key: str) -> None: + dict.__delitem__(self, key) + del self.parent[f"{self.namespace}.{key}"] + # Collection utilities diff --git a/trulens_eval/trulens_eval/utils/pyschema.py b/trulens_eval/trulens_eval/utils/pyschema.py index 76f489920..a19a0295d 100644 --- a/trulens_eval/trulens_eval/utils/pyschema.py +++ b/trulens_eval/trulens_eval/utils/pyschema.py @@ -146,6 +146,9 @@ class Module(SerialModel): package_name: Optional[str] = None # some modules are not in a package module_name: str + def __hash__(self): + return hash((self.package_name, self.module_name)) + def of_module(mod: ModuleType, loadable: bool = False) -> 'Module': if loadable and mod.__name__ == "__main__": # running in notebook @@ -185,6 +188,9 @@ class Class(SerialModel): bases: Optional[Sequence[Class]] = None + def __hash__(self): + return hash((self.name, self.module)) + def __repr__(self): return self.module.module_name + "." + self.name @@ -495,6 +501,15 @@ class Method(FunctionOrMethod): obj: Obj name: str + def __hash__(self): + return hash((self.obj.cls, self.name)) + + def as_func(self) -> Function: + """View self as a function instead of method, with the function + stripping away object information.""" + + return Function(cls=self.obj.cls, module=self.obj.cls.module, name=self.name) + @staticmethod def of_method( meth: Callable, @@ -546,6 +561,9 @@ class Function(FunctionOrMethod): name: str + def __hash__(self): + return hash((self.module, self.cls, self.name)) + @staticmethod def of_function( func: Callable, diff --git a/trulens_eval/trulens_eval/utils/python.py b/trulens_eval/trulens_eval/utils/python.py index 339bd1db2..c75641c26 100644 --- a/trulens_eval/trulens_eval/utils/python.py +++ b/trulens_eval/trulens_eval/utils/python.py @@ -267,7 +267,6 @@ def code_line(func, show_source: bool = False) -> Optional[str]: ret += "\t" + str(line) return ret - def locals_except(*exceptions):