[DRAFT] generators debugging

.azure_pipelines/ci-eval-pr.yaml

@@ -1,3 +1,6 @@
+# This a definition for azure pipelines, not github pipelines. There are
+# differences between these systems.
+
 pr:
   branches:
     include:
@@ -6,47 +9,133 @@ pr:
   paths:
     include:
       - trulens_eval
-      - docs/trulens_eval/*.ipynb
-
 
 jobs:
-  - job: PRBranchProtect
+  - job: PRBranchProtect # name seems to be important but cannot figure out why
     pool:
-      vmImage: "ubuntu-latest"
+      vmImage: ubuntu-latest
     timeoutInMinutes: 30
 
     strategy:
       matrix:
-        default:
-          condaEnvFileSuffix: "trulens-llm"
+        default: # name is important
+          python-version: 3.11
+          tests-folder: tests/unit
+        py308-static:
+          python-version: 3.8
+          tests-folder: tests/unit/static
+        py309-static:
+          python-version: 3.9
+          tests-folder: tests/unit/static
+        py310-static:
+          python-version: 3.10
+          tests-folder: tests/unit/static
+
+    # This doesn't seem to be neccessary, the env is setup without it perhaps by
+    # Azure pipeline config?
+    #env:
+    #  OPENAI_API_KEY: $(OPENAI_API_KEY)
+    #  HUGGINGFACE_API_KEY: $(HUGGINGFACE_API_KEY)
+    #  PINECONE_API_KEY: $(PINECONE_API_KEY)
+    #  PINECONE_ENV: $(PINECONE_ENV)
+    #  HUGGINGFACEHUB_API_TOKEN: $(HUGGINGFACEHUB_API_TOKEN)
 
     steps:
       - checkout: self
         clean: true
-      - bash: echo "##vso[task.prependpath]$CONDA/bin"
-        displayName: Add conda to PATH
+
       - bash: |
+          echo "##vso[task.prependpath]$CONDA/bin"
+
           set -e
-          conda create  -y --quiet -n $(condaEnvFileSuffix) python=3
+          conda create -y --quiet \
+            -n $(python-version) \
+            python=$(python-version)
+
           conda init bash
-        displayName: Create Anaconda environment
-      - bash: |
-          set -e
-          source activate $(condaEnvFileSuffix)
-          which python
-        displayName: Which Python
+
+        displayName: Setup conda
+
       - bash: |
           set -e
-          source activate $(condaEnvFileSuffix)
+          source activate $(python-version)
+
           pip install yapf==0.32.0
-          ./format.sh
-        displayName: Format Code
-      - bash: |
-          num_changed_files=`git ls-files --others -m --exclude-standard | wc -l`
+          pip install isort==5.10.1
+
+          yapf --version
+          isort --vn
+
+          ./format.sh --eval
+
+          num_changed_files=`git ls-files --others -m --exclude-standard ./trulens_eval | wc -l`
           if [ $num_changed_files -ne 0 ]; then
             echo "The following files have changed after running format.sh. Please format your code and update the PR."
-            git ls-files --others -m --exclude-standard
+            git ls-files --others -m --exclude-standard ./trulens_eval
             echo "\n\nOutputting git diff for checked in files.\n"
             git diff
           fi
-        displayName: NoDiffCheck
+        condition: eq(variables['python-version'], 3.11) # only run format check with latest python version
+        displayName: Format diffs
+
+      - bash: |
+          set -e
+          source activate $(python-version)
+
+          cd ./trulens_eval
+          pip install -e .
+
+          python --version
+          pip --version
+          pip list
+        displayName: Install trulens
+
+      - bash: |
+          set -e
+          source activate $(python-version)
+
+          pip install \
+            pytest==7.0.1 pytest-subtests pytest-azurepipelines
+
+        displayName: Install testing packages
+
+      - bash: |
+          set -e
+          source activate $(python-version)
+
+          cd ./trulens_eval
+          python -m pytest --test-run-title="Required $(python-version) unit tests" $(tests-folder)
+        displayName: Unit tests with required packages
+
+      - bash: |
+          set -e
+          source activate $(python-version)
+
+          cd ./trulens_eval
+
+          # Once we start running notebooks in these tests, might have to do this again:
+          # pip install ipykernel
+          # python -m ipykernel install --user \
+          #  --name $(python-version) \
+          #  --display-name $(python-version)
+
+          pip install -r trulens_eval/requirements.optional.txt
+
+          python --version
+          pip --version
+          pip list
+        displayName: Install optional packages
+
+      - bash: |
+          set -e
+          source activate $(python-version)
+
+          cd ./trulens_eval
+
+          python -m pytest  --test-run-title="Optional $(python-version) unit tests" $(tests-folder)
+        env:
+          # enables optional tests, see tests/unit/test.py
+          TEST_OPTIONAL: true 
+
+        displayName: Unit tests with optional packages
+

.azure_pipelines/ci-eval.yaml

@@ -16,6 +16,12 @@ pr:
       - trulens_eval
       - docs/trulens_eval/*.ipynb
 
+schedules:
+  - cron: "0 0 * * *"
+    displayName: Daily midnight build
+    branches:
+      include:
+        - main
 
 jobs:
   - job: quickstartTests
@@ -33,59 +39,79 @@ jobs:
     steps:
       - checkout: self
         clean: true
+
       - bash: echo "##vso[task.prependpath]$CONDA/bin"
         displayName: Add conda to PATH
+
       - bash: |
           set -e
-          conda create  -y --quiet -n $(condaEnvFileSuffix) python=3
+          conda create  -y --quiet -n $(condaEnvFileSuffix) python=3.11
           conda init bash
         displayName: Create Anaconda environment
+
       - bash: |
           set -e
           source activate $(condaEnvFileSuffix)
           which python
+          python --version
         displayName: Which Python
+
       - bash: |
           set -e
           source activate $(condaEnvFileSuffix)
           cd ./trulens_eval
           pip install -e .
           pip install -r $(requirementsFile)
           pip install yapf==0.32.0
+          pip install isort==5.10.1
           pip install ipykernel
           python -m ipykernel install --user --name $(condaEnvFileSuffix) --display-name $(condaEnvFileSuffix)
         displayName: Install pip package
+
       - bash: |
           set -e
           source activate $(condaEnvFileSuffix)
-          ./format.sh
+          echo Using $(yapf --version) isort $(isort --vn)
+          ./format.sh --eval
         displayName: Format Code
+
       - bash: |
-          num_changed_files=`git ls-files --others -m --exclude-standard | wc -l`
+          num_changed_files=`git ls-files --others -m --exclude-standard ./trulens_eval | wc -l`
           if [ $num_changed_files -ne 0 ]; then
             echo "The following files have changed after running format.sh. Please format your code and update the PR."
-            git ls-files --others -m --exclude-standard
+            git ls-files --others -m --exclude-standard ./trulens_eval
             echo "\n\nOutputting git diff for checked in files.\n"
             git diff
           fi
         displayName: NoDiffCheck
+
       - bash: |
           source activate $(condaEnvFileSuffix)
           cd ./docs/trulens_eval
           pip install jupyter
           pip install nbmerge
           ../../.github/workflows/combine_nb_to_docs_testing.sh
         displayName: Run combine docs for testing
+
+      - bash: |
+          source activate $(condaEnvFileSuffix)
+
+          cd ./trulens_eval
+          pip install pytest==7.0.1 pytest-azurepipelines
+          pip install -r trulens_eval/requirements.optional.txt
+        displayName: Install optional deps
+
       - bash: |
           source activate $(condaEnvFileSuffix)
-          pip install pytest==7.0.1 pytest-azurepipelines langchain>=0.0.170 llama_index>=0.6.24 
-          pip install docarray hnswlib
-          echo '::group::piplist'
           echo "$(pip list)"
-          echo '::endgroup::'
+        displayName: Pip list
+
+      - bash: |
+          source activate $(condaEnvFileSuffix)
           cd ./trulens_eval
+
           python -m pytest $(testSubdirectory)
-        displayName: Run unit tests
+        displayName: Run notebook tests
         env:
           OPENAI_API_KEY: $(OPENAI_API_KEY)
           HUGGINGFACE_API_KEY: $(HUGGINGFACE_API_KEY)

.azure_pipelines/ci.yaml

@@ -22,33 +22,33 @@ jobs:
 
     strategy:
       matrix:
-        py37_tf1_keras:
-          condaEnvFileSuffix: "python37"
+        py38_tf1_keras:
+          condaEnvFileSuffix: "python38"
           requirementsFile: "tests/keras/requirements_tfv1.txt"
           testSubdirectory: "tests/keras"
 
-        py37_tf1_tf_keras:
-          condaEnvFileSuffix: "python37"
+        py38_tf1_tf_keras:
+          condaEnvFileSuffix: "python38"
           requirementsFile: "tests/tf_keras/requirements_tfv1.txt"
           testSubdirectory: "tests/tf_keras"
 
-        py37_pytorch:
-          condaEnvFileSuffix: "python37"
+        py38_pytorch:
+          condaEnvFileSuffix: "python38"
           requirementsFile: "tests/pytorch/requirements.txt"
           testSubdirectory: "tests/pytorch"
 
-        py37_tf1:
-          condaEnvFileSuffix: "python37"
+        py38_tf1:
+          condaEnvFileSuffix: "python38"
           requirementsFile: "tests/tf/requirements.txt"
           testSubdirectory: "tests/tf"
 
-        py37_tf2:
-          condaEnvFileSuffix: "python37"
+        py38_tf2:
+          condaEnvFileSuffix: "python38"
           requirementsFile: "tests/tf2/requirements.txt"
           testSubdirectory: "tests/tf2"
 
-        py37_tf2_non_eager:
-          condaEnvFileSuffix: "python37"
+        py38_tf2_non_eager:
+          condaEnvFileSuffix: "python38"
           requirementsFile: "tests/tf2/requirements.txt"
           testSubdirectory: "tests/tf2_non_eager"
 
@@ -79,6 +79,7 @@ jobs:
           set -e
           source activate $(condaEnvFileSuffix)
           which python
+          python --version
         displayName: Which Python
       - bash: |
           set -e
@@ -92,13 +93,14 @@ jobs:
       - bash: |
           set -e
           source activate $(condaEnvFileSuffix)
-          ./format.sh
+          echo Using $(yapf --version) isort $(isort --vn)
+          ./format.sh --explain
         displayName: Format Code
       - bash: |
-          num_changed_files=`git ls-files --others -m --exclude-standard | wc -l`
+          num_changed_files=`git ls-files --others -m --exclude-standard ./trulens_explain | wc -l`
           if [ $num_changed_files -ne 0 ]; then
             echo "The following files have changed after running format.sh. Please format your code and update the PR."
-            git ls-files --others -m --exclude-standard
+            git ls-files --others -m --exclude-standard ./trulens_explain
             echo "\n\nOutputting git diff for checked in files.\n"
             git diff
           fi