Merge branch 'main' into minor-change

.github/workflows/build-python.yml

@@ -103,11 +103,6 @@ jobs:
           python-version: ${{ matrix.python-version }}
           cache: true
 
-      - name: Set up Pandoc (needed for doc)
-        uses: r-lib/actions/setup-pandoc@v2
-        with:
-          pandoc-version: '3.1.7' # https://github.com/jgm/pandoc/releases
-
       - name: Cache Giskard test resources
         uses: actions/cache@v3
         if: ${{ github.event_name == 'pull_request' || inputs.use-cache }}
@@ -160,22 +155,6 @@ jobs:
           pdm run pip freeze | grep '^pandas'
           pdm run pip freeze | grep -q '^pandas==${{ matrix.pandas_v1 && '1' || '2' }}\.'
 
-      - name: Test code (RAM expensive in tests/models, tests/testing, tests/test_metamorphic_invariance.py)
-        if: ${{ startsWith(matrix.os, 'windows') }}
-        run: pdm test-ram tests/models tests/testing tests/test_metamorphic_invariance.py
-
-      - name: Test code (RAM expensive, /tests/models and /tests/testing)
-        if: ${{ !startsWith(matrix.os, 'windows') }}
-        run: pdm test-ram --memray tests/models tests/testing tests/test_metamorphic_invariance.py
-
-      - name: Test code (RAM expensive, others)
-        if: ${{ startsWith(matrix.os, 'windows') }}
-        run: pdm test-ram tests/ --ignore=tests/models  --ignore=tests/testing --ignore=tests/test_metamorphic_invariance.py
-
-      - name: Test code (RAM expensive, others)
-        if: ${{ !startsWith(matrix.os, 'windows') }}
-        run: pdm test-ram --memray tests/ --ignore=tests/models --ignore=tests/testing --ignore=tests/test_metamorphic_invariance.py
-
       - name: Test code (concurrency)
         run: pdm test-worker
 
@@ -191,10 +170,6 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
 
-      - name: Build doc
-        if : ${{ matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10'}}
-        run: pdm run doc
-
       - name: Build
         run: pdm build
 
@@ -208,13 +183,7 @@ jobs:
         if: ${{ inputs.run-integration-tests }}
         env:
           PYTEST_XDIST_AUTO_NUM_WORKERS: 2
-        run: pdm test-slow tests/integrations
-
-      - name: Run integration tests for python
-        if: ${{ inputs.run-integration-tests }}
-        env:
-          PYTEST_XDIST_AUTO_NUM_WORKERS: 2
-        run: pdm test-slow tests/ --ignore=tests/integrations
+        run: pdm test-slow tests/
 
       - name: "Memory csv"
         if: ${{ always() && matrix.os == 'ubuntu-latest' && matrix.python-version == '3.10' && !matrix.langchain_minimal && !matrix.pandas_v1 && !matrix.pydantic_v1 }}
@@ -300,3 +269,31 @@ jobs:
           sed -i 's/^\(requires-python *= *\).*$/\1">=3.10,<3.12"/' pyproject.toml
           pdm add "$(ls ../dist/*.whl)"
           pdm run python -c "import giskard"
+  check-doc:
+    name: "Build and check doc"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/[email protected]
+        with:
+          fetch-depth: 0
+
+      - name: Setup PDM
+        uses: pdm-project/setup-pdm@v3
+        with:
+          python-version: "3.10"
+          cache: true
+
+      - name: Set up Pandoc (needed for doc)
+        uses: r-lib/actions/setup-pandoc@v2
+        with:
+          pandoc-version: '3.1.7' # https://github.com/jgm/pandoc/releases
+
+      - name: Install dependencies
+        run: pdm install -G :all
+
+      - name: Build doc
+        run: pdm  doc
+
+      - name: Check doc
+        run: pdm check-doc

conftest.py

@@ -1,11 +1,19 @@
+from typing import List
+
 import gc
+import json
 import os
+import subprocess
+import sys
 import time
 from pathlib import Path
+from tempfile import NamedTemporaryFile
 
 import psutil
 import pytest
+from _pytest.config.argparsing import Parser
 from _pytest.python import Function
+from _pytest.reports import TestReport
 
 
 @pytest.hookimpl(hookwrapper=True)
@@ -60,3 +68,43 @@ def pytest_runtest_protocol(item: Function, nextitem: Function):
 
         # Add overall test results
         writer.write(f"{item.nodeid},{test_memory_usage:.3f},{full_memory_usage:.3f}\n")
+
+
+# we know this bit is bad, but we cant help it with the current pytest setup
+def pytest_addoption(parser: Parser):
+    parser.addoption("--use-subprocess", action="store_true", default=False, help="Whether to use subprocess")
+
+
+def separate_process(item: Function) -> List[TestReport]:
+    with NamedTemporaryFile(delete=False) as fp:
+        proc = subprocess.run(
+            shell=True,
+            check=False,
+            stdout=sys.stdout,
+            stderr=sys.stderr,
+            args=f"{sys.executable} -m pytest {item.nodeid} -vvv --tb=long --report-log={fp.name} --no-header --no-summary",
+            cwd=Path(__file__).parent,
+        )
+
+    reports = []
+    try:
+        for line in Path(fp.name).read_text().splitlines():
+            report_dict = json.loads(line)
+            if report_dict["$report_type"] == "TestReport":
+                reports.append(TestReport._from_json(report_dict))
+        return reports
+    finally:
+        # Force deletion of the temp file
+        Path(fp.name).unlink(missing_ok=True)
+
+
+# https://docs.pytest.org/en/7.1.x/reference/reference.html#pytest.hookspec.pytest_runtest_protocol
+@pytest.hookimpl(tryfirst=True)
+def pytest_runtest_call(item: Function):
+    mark = item.get_closest_marker("skip")
+    skip = mark is not None
+    mark = item.get_closest_marker("skipif")
+    skip |= mark is not None and ((len(mark.args) == 1 and mark.args[0]) or mark.kwargs.get("condition", False))
+    if not skip and item.get_closest_marker("memory_expensive") and item.config.getoption("--use-subprocess"):
+        reports = separate_process(item)
+        return reports

docs/cli/ngrok/index.rst

@@ -1,27 +1,27 @@
 Setup a :code:`ngrok` account
-======================
+=============================
 
 In order to expose the Giskard Hub to the internet, you would need to perform the following steps
 
-1. Sign up `here <https://dashboard.ngrok.com/signup>`_
-^^^^^^^^^^^^^^^^^^^^^^^
+1. Sign up `here <https://dashboard.ngrok.com/signup>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 You will be prompted by the following:
 
 .. image:: ../../assets/ngrok_aut.png
   :width: 400
 
 You would need to have either :code:`Google Authenticator` or :code:`1Password` on your phone to generate codes.
 
-2. Generate an API key `here <https://dashboard.ngrok.com/get-started/your-authtoken>`_
-^^^^^^^^^^^^^^^^^^^^^^^
+2. Generate an API key `here <https://dashboard.ngrok.com/get-started/your-authtoken>`__
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Copy the following key:
 
 .. image:: ../../assets/ngrok_aut2.png
   :width: 400
 
 
 3. Expose the giskard hub
-^^^^^^^^^^^^^^^^^^^^^^^
+^^^^^^^^^^^^^^^^^^^^^^^^^
 Now you can run :code:`giskard hub expose --ngrok-token <ngrok_API_key>` which should prompt you with the following instructions:::
 
     Exposing Giskard Hub to the internet...

docs/community/contribution_guidelines/dev-environment.md

@@ -2,7 +2,7 @@
 
 After cloning the repository here are the useful commands to set up the local environment:
 
-### Pre-commit hooks installation
+## Pre-commit hooks installation
 Pre-commit hooks ensure that the basic code quality checks are performed before the code is committed
 
 The current pre-commit hool configuration is defined in `.pre-commit-config.yaml`

docs/community/contribution_guidelines/index.rst

@@ -21,15 +21,15 @@ Push your tests in the Giskard repo
 * Clone the Giskard repository
 * Create a GitHub branch with the base as main, starting with `test-contribution/name-of-your-branch`
 * From the root of the cloned repo run `./gradlew generateProto`. This will generate the module`generated` that you will need to create your tests.
-* Write your test inside one of the classes (`MetamorphicTests`, `HeuristicTests`, `PerformanceTests` or `DriftTests`) inside this `repo <https://github.com/Giskard-AI/giskard/tree/main/giskard-ml-worker/ml\_worker/testing>`_. If your test does not fit these classes, you can also create a custom class in a new file.
-* We recommend writing unit tests for your test functions: this is the way you can execute and debug your test! Unit tests should be placed in `this directory <https://github.com/Giskard-AI/giskard/tree/main/giskard-ml-worker/test>`_.
+* Write your test inside one of the classes (`MetamorphicTests`, `HeuristicTests`, `PerformanceTests` or `DriftTests`) inside this `repo <https://github.com/Giskard-AI/giskard/tree/main/giskard-ml-worker/ml\_worker/testing>`__. If your test does not fit these classes, you can also create a custom class in a new file.
+* We recommend writing unit tests for your test functions: this is the way you can execute and debug your test! Unit tests should be placed in `this directory <https://github.com/Giskard-AI/giskard/tree/main/giskard-ml-worker/test>`__.
 
 .. hint::
     **Fixtures**
 
     A unit test is executed with a *test model* and *test data* provided as fixtures.
 
-    For example, in `test_precision <https://github.com/Giskard-AI/giskard/blob/main/giskard-ml-worker/test/test\_performance.py#L73>`_ function, we use *german_credit_data* and *german_credit_model* as fixtures. If necessary, you can create your own fixtures; check `this directory <https://github.com/Giskard-AI/giskard/tree/main/giskard-ml-worker/test/fixtures>`_ to know how.
+    For example, in `test_precision <https://github.com/Giskard-AI/giskard/blob/main/giskard-ml-worker/test/test\_performance.py#L73>`_ function, we use *german_credit_data* and *german_credit_model* as fixtures. If necessary, you can create your own fixtures; check `this directory <https://github.com/Giskard-AI/giskard/tree/main/giskard-ml-worker/test/fixtures>`__ to know how.
 
 * Create a Pull Request
 

docs/conf.py

@@ -140,26 +140,39 @@ def linkcode_resolve(domain, info):
 
     modname = info["module"]
     fullname = info["fullname"]
+    print("##############")
+    print(f"modname:{modname}")
+    print(f"fullname:{fullname}")
 
     submod = sys.modules.get(modname)
-    print(submod)
+    # print(submod)
     if submod is None:
-        return None
+        print("##############")
 
+        return None
+    obj = submod
     for part in fullname.split("."):
         try:
-            obj = getattr(submod, part)
-            print(obj)
+            obj = getattr(obj, part)
+            print(f"obj:{obj}")
+
+            # print(obj)
         except:  # noqa: E722
+            print("##############")
             return None
 
     try:
-        fn = inspect.getsourcefile(obj.test_fn)  # TODO: generalise for other objects!
-        print(fn)
+        fn = inspect.getsourcefile(
+            obj.test_fn if hasattr(obj, "test_fn") else obj
+        )  # TODO: generalise for other objects!
+    # print(fn)
     except:  # noqa: E722
         fn = None
     if not fn:
+        print("##############")
+
         return None
+    print(f"fn:{fn}")
 
     try:
         source, lineno = inspect.getsourcelines(obj)
@@ -170,6 +183,9 @@ def linkcode_resolve(domain, info):
         linespec = "#L%d-L%d" % (lineno, lineno + len(source) - 1)
     else:
         linespec = ""
+    print(f"linespec:{linespec}")
+
+    filename = fn.split("giskard")[-1]
+    print("##############")
 
-    filename = fn.split("main", 1)[-1]
-    return f"https://github.com/Giskard-AI/giskard/blob/main/{filename}{linespec}"
+    return f"https://github.com/Giskard-AI/giskard/blob/main/giskard{filename}{linespec}"