make format

trulens_eval/Makefile

@@ -57,7 +57,7 @@ test-tru-custom:
 
 
 format:
-	$(CONDA); bash ../format.sh
+	$(CONDA); cd ..; bash format.sh --eval
 
 lab:
 	$(CONDA); jupyter lab --ip=0.0.0.0 --no-browser --ServerApp.token=deadbeef

trulens_eval/examples/expositional/end2end_apps/custom_app/custom_app.py

@@ -1,6 +1,6 @@
 import asyncio
-import time
 from concurrent.futures import wait
+import time
 
 from examples.expositional.end2end_apps.custom_app.custom_llm import CustomLLM
 from examples.expositional.end2end_apps.custom_app.custom_memory import \
@@ -30,8 +30,8 @@ def fill(self, question, answer):
 class CustomApp:
 
     def __init__(self, delay: float = 0.05, alloc: int = 1024 * 1024):
-        self.delay = delay # controls how long to delay certain operations to make it look more realistic
-        self.alloc = alloc # controls how much memory to allocate during some operations
+        self.delay = delay  # controls how long to delay certain operations to make it look more realistic
+        self.alloc = alloc  # controls how much memory to allocate during some operations
         self.memory = CustomMemory(delay=delay, alloc=alloc)
         self.retriever = CustomRetriever(delay=delay, alloc=alloc)
         self.llm = CustomLLM(delay=delay, alloc=alloc)

trulens_eval/examples/expositional/end2end_apps/custom_app/custom_llm.py

@@ -1,12 +1,17 @@
-import time
 import sys
+import time
 
 from trulens_eval.tru_custom_app import instrument
 
 
 class CustomLLM:
 
-    def __init__(self, model: str = "derp", delay: float = 0.01, alloc: int = 1024 * 1024):
+    def __init__(
+        self,
+        model: str = "derp",
+        delay: float = 0.01,
+        alloc: int = 1024 * 1024
+    ):
         self.model = model
         self.delay = delay
         self.alloc = alloc
@@ -18,4 +23,5 @@ def generate(self, prompt: str):
 
         temporary = [0x42] * self.alloc
 
-        return "herp " + prompt[::-1] + f" derp and {sys.getsizeof(temporary)} bytes"
+        return "herp " + prompt[::-1
+                               ] + f" derp and {sys.getsizeof(temporary)} bytes"

trulens_eval/examples/expositional/end2end_apps/custom_app/custom_memory.py

@@ -1,5 +1,5 @@
-import time
 import sys
+import time
 
 from trulens_eval.tru_custom_app import instrument
 
@@ -20,5 +20,6 @@ def remember(self, data: str):
             time.sleep(self.delay)
 
         self.messages.append(
-            data + f" and I'm keeping around {sys.getsizeof(self.temporary)} bytes"
+            data +
+            f" and I'm keeping around {sys.getsizeof(self.temporary)} bytes"
         )

trulens_eval/examples/expositional/end2end_apps/custom_app/custom_retriever.py

@@ -1,5 +1,5 @@
-import time
 import sys
+import time
 
 from trulens_eval.tru_custom_app import instrument
 
@@ -18,7 +18,6 @@ def retrieve_chunks(self, data):
             time.sleep(self.delay)
 
         return [
-            f"Relevant chunk: {data.upper()}",
-            f"Relevant chunk: {data[::-1]}",
+            f"Relevant chunk: {data.upper()}", f"Relevant chunk: {data[::-1]}",
             f"Relevant chunk: I allocated {sys.getsizeof(temporary)} bytes to pretend I'm doing something."
         ]

trulens_eval/trulens_eval/app.py

@@ -12,8 +12,10 @@
 import logging
 from pprint import PrettyPrinter
 from threading import Lock
-from typing import (Any, Callable, ClassVar, Dict, Hashable, Iterable, List,
-                    Optional, Sequence, Set, Tuple, Type)
+from typing import (
+    Any, Callable, ClassVar, Dict, Hashable, Iterable, List, Optional, Sequence,
+    Set, Tuple, Type
+)
 
 import pydantic
 from pydantic import Field

trulens_eval/trulens_eval/database/sqlalchemy_db.py

@@ -2,19 +2,20 @@
 from datetime import datetime
 import json
 import logging
-from typing import (Any, ClassVar, Dict, Iterable, List, Optional, Sequence, Tuple,
-                    Union)
+from typing import (
+    Any, ClassVar, Dict, Iterable, List, Optional, Sequence, Tuple, Union
+)
 import warnings
 
 import numpy as np
 import pandas as pd
 from pydantic import Field
 from sqlalchemy import create_engine
 from sqlalchemy import Engine
+from sqlalchemy import func
 from sqlalchemy import select
 from sqlalchemy.orm import sessionmaker
 from sqlalchemy.schema import MetaData
-from sqlalchemy import func
 
 from trulens_eval import schema
 from trulens_eval.database import orm
@@ -279,7 +280,7 @@ def _feedback_query(
             q = func.count(orm.FeedbackResult.feedback_result_id)
         else:
             q = select(orm.FeedbackResult)
-        
+
         if record_id:
             q = q.filter_by(record_id=record_id)
 
@@ -329,15 +330,14 @@ def get_feedback_count_by_status(
 
         with self.Session.begin() as session:
             q = self._feedback_query(
-                count=True,
-                **locals_except("self", "session")
+                count=True, **locals_except("self", "session")
             )
 
-            results = session.query(orm.FeedbackResult.status, q).group_by(orm.FeedbackResult.status)
+            results = session.query(orm.FeedbackResult.status,
+                                    q).group_by(orm.FeedbackResult.status)
 
             return {FeedbackResultStatus(row[0]): row[1] for row in results}
 
-
     def get_feedback(
         self,
         record_id: Optional[RecordID] = None,
@@ -398,7 +398,7 @@ def _extract(_result: orm.FeedbackResult):
             _result.name,
             _result.result,
             _result.multi_result,
-            _result.cost_json, # why is cost_json not parsed?
+            _result.cost_json,  # why is cost_json not parsed?
             json.loads(_result.record.perf_json)
             if _result.record.perf_json != MIGRATION_UNKNOWN_STR else no_perf,
             json.loads(_result.calls_json)["calls"],

trulens_eval/trulens_eval/feedback/feedback.py

@@ -240,10 +240,7 @@ def evaluate_deferred(
         tru: 'Tru',
         limit: Optional[int] = None,
         shuffle: bool = False
-    ) -> List[Tuple[
-        pandas.Series,
-        Future[FeedbackResult]
-    ]]:
+    ) -> List[Tuple[pandas.Series, Future[FeedbackResult]]]:
         """
         Evaluates feedback functions that were specified to be deferred. Returns
         a list of tuples with the DB row containing the Feedback and initial
@@ -292,12 +289,11 @@ def prepare_feedback(row) -> Optional[FeedbackResultStatus]:
         # Get the different status feedbacks except those marked DONE.
         feedbacks_not_done = db.get_feedback(
             status=[
-                FeedbackResultStatus.NONE,
-                FeedbackResultStatus.FAILED,
+                FeedbackResultStatus.NONE, FeedbackResultStatus.FAILED,
                 FeedbackResultStatus.RUNNING
             ],
-            limit = limit,
-            shuffle = shuffle,
+            limit=limit,
+            shuffle=shuffle,
         )
 
         tp = TP()

trulens_eval/trulens_eval/schema.py

@@ -28,8 +28,8 @@
 import logging
 from pprint import PrettyPrinter
 from typing import (
-    Any, Callable, ClassVar, Dict, Hashable, List, Optional, Sequence, Tuple, Type,
-    TYPE_CHECKING, TypeVar, Union
+    Any, Callable, ClassVar, Dict, Hashable, List, Optional, Sequence, Tuple,
+    Type, TYPE_CHECKING, TypeVar, Union
 )
 
 import dill
@@ -218,7 +218,9 @@ def __init__(self, record_id: Optional[RecordID] = None, **kwargs):
     def __hash__(self):
         return hash(self.record_id)
 
-    def wait_for_feedback_results(self) -> Dict[FeedbackDefinition, FeedbackResult]:
+    def wait_for_feedback_results(
+        self
+    ) -> Dict[FeedbackDefinition, FeedbackResult]:
         """
         Wait for feedback results to finish and return a mapping of feedback
         functions to their results.

trulens_eval/trulens_eval/tru.py

@@ -14,8 +14,9 @@
 import threading
 from threading import Thread
 from time import sleep
-from typing import (Callable, Dict, Iterable, List, Optional, Sequence, Tuple,
-                    Union)
+from typing import (
+    Callable, Dict, Iterable, List, Optional, Sequence, Tuple, Union
+)
 import warnings
 
 import humanize
@@ -50,9 +51,11 @@
 # dashboard.
 DASHBOARD_START_TIMEOUT = 30
 
+
 def humanize_seconds(seconds: float):
     return humanize.naturaldelta(timedelta(seconds=seconds))
 
+
 class Tru(SingletonPerName):
     """
     Tru is the main class that provides an entry points to trulens-eval. Tru lets you:
@@ -85,7 +88,6 @@ class Tru(SingletonPerName):
     # Process of the dashboard app.
     dashboard_proc = None
 
-
     def Chain(__tru_self, chain, **kwargs):
         """
         Create a TruChain with database managed by self.
@@ -369,11 +371,9 @@ def get_leaderboard(self, app_ids: List[str]):
 
         return leaderboard
 
-    def start_evaluator(
-        self,
-        restart: bool = False,
-        fork: bool = False
-    ) -> Union[Process, Thread]:
+    def start_evaluator(self,
+                        restart: bool = False,
+                        fork: bool = False) -> Union[Process, Thread]:
         """
         Start a deferred feedback function evaluation thread.
 
@@ -413,10 +413,12 @@ def runloop():
 
             print(
                 f"Will keep max of "
-                f"{self.DEFERRED_NUM_RUNS} feedback(s) running.")
+                f"{self.DEFERRED_NUM_RUNS} feedback(s) running."
+            )
             print(
                 f"Tasks are spread among max of "
-                f"{TP.MAX_THREADS} thread(s).")
+                f"{TP.MAX_THREADS} thread(s)."
+            )
             print(
                 f"Will rerun running feedbacks after "
                 f"{humanize_seconds(self.RETRY_RUNNING_SECONDS)}."
@@ -429,7 +431,9 @@ def runloop():
             total = 0
 
             # Show the overall counts from the database, not just what has been looked at so far.
-            tqdm_status = tqdm(desc="Feedback Status", initial=0, unit="feedbacks")
+            tqdm_status = tqdm(
+                desc="Feedback Status", initial=0, unit="feedbacks"
+            )
 
             # Show the status of the results so far.
             tqdm_total = tqdm(desc="Done Runs", initial=0, unit="runs")
@@ -510,7 +514,10 @@ def runloop():
                 tqdm_status.n = queue_done
                 tqdm_status.total = queue_total
                 tqdm_status.set_postfix(
-                    {status.name: count for status, count in queue_stats.items()}
+                    {
+                        status.name: count
+                        for status, count in queue_stats.items()
+                    }
                 )
 
                 # Check if any of the running futures should be stopped.
@@ -522,7 +529,7 @@ def runloop():
                         # Not checking status here as this will be not yet be set
                         # correctly. The computation in the future update the
                         # database but this object is outdated.
-                        
+
                         elapsed = datetime.now().timestamp() - row.last_ts
                         if elapsed > self.RETRY_RUNNING_SECONDS:
                             fut.cancel()