draft validation functions (#90)

* draft validation functions

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* run on all atlases, don't crash on assertion error

* fixing atlas path

* Clearer output printing

* tidy up validation script, remove weird test_git

* add dev install, make test structure, initial tests

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* add tests and return for _assert_close()

* add test for validate mesh matches annotation

* fix linting

* update version for actions

* drop py3.8 in tox, run pytest in tox

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix copy-paste error in pytest command

* drop py3.8 from gh action workflow file too

* Adding docstrings to validation script

* Making path tests stricter, breaking up long strings, adding diff_tolerance argument to _assert_close function

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* restructuring validate_mesh_matches_image_extents function, adding comments

* testing expected files and  meshes directory separately

* looping through validation functions and parameters to catch individual errors

* removing hard coded path, generalising to all atlases

* adding successful_validations list

* tidying up duplications

* fix recursive bug

* addressing Niko's final comments, cleaning code

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Co-authored-by: Alessandro Felder <[email protected]>
Co-authored-by: alessandrofelder <[email protected]>

.github/workflows/test_and_deploy.yml

@@ -9,7 +9,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: neuroinformatics-unit/actions/lint@v1
+      - uses: neuroinformatics-unit/actions/lint@v2
 
   test:
     needs: lint
@@ -25,10 +25,8 @@ jobs:
           python-version: "3.10"
         - os: windows-latest
           python-version: "3.9"
-        - os: ubuntu-latest
-          python-version: "3.8"
 
     steps:
-      - uses: neuroinformatics-unit/actions/test@v1
+      - uses: neuroinformatics-unit/actions/test@v2
         with:
           python-version: ${{ matrix.python-version }}

bg_atlasgen/validate_atlases.py

@@ -0,0 +1,168 @@
+"""Script to validate atlases"""
+
+
+from pathlib import Path
+
+import numpy as np
+from bg_atlasapi import BrainGlobeAtlas
+from bg_atlasapi.config import get_brainglobe_dir
+from bg_atlasapi.list_atlases import (
+    get_all_atlases_lastversions,
+    get_atlases_lastversions,
+)
+from bg_atlasapi.update_atlases import update_atlas
+
+
+def validate_atlas_files(atlas_path: Path):
+    """Checks if basic files exist in the atlas folder"""
+
+    assert atlas_path.is_dir(), f"Atlas path {atlas_path} not found"
+    expected_files = [
+        "annotation.tiff",
+        "reference.tiff",
+        "metadata.json",
+        "structures.json",
+    ]
+    for expected_file_name in expected_files:
+        expected_path = Path(atlas_path / expected_file_name)
+        assert (
+            expected_path.is_file()
+        ), f"Expected file not found at {expected_path}"
+
+    meshes_path = atlas_path / "meshes"
+    assert meshes_path.is_dir(), f"Meshes path {meshes_path} not found"
+    return True
+
+
+def _assert_close(mesh_coord, annotation_coord, pixel_size, diff_tolerance=10):
+    """
+    Helper function to check if the mesh and the annotation coordinate
+    are closer to each other than an arbitrary tolerance value times the pixel size.
+    The default tolerance value is 10.
+    """
+    assert abs(mesh_coord - annotation_coord) <= diff_tolerance * pixel_size, (
+        f"Mesh coordinate {mesh_coord} and annotation coordinate {annotation_coord}",
+        f"differ by more than {diff_tolerance} times pixel size {pixel_size}",
+    )
+    return True
+
+
+def validate_mesh_matches_image_extents(atlas: BrainGlobeAtlas):
+    """Checks if the mesh and the image extents are similar"""
+
+    root_mesh = atlas.mesh_from_structure("root")
+    annotation_image = atlas.annotation
+    resolution = atlas.resolution
+
+    # minimum and maximum values of the annotation image (z, y, x)
+    z_range, y_range, x_range = np.nonzero(annotation_image)
+    z_min, z_max = np.min(z_range), np.max(z_range)
+    y_min, y_max = np.min(y_range), np.max(y_range)
+    x_min, x_max = np.min(x_range), np.max(x_range)
+
+    # minimum and maximum values of the annotation image scaled by the atlas resolution
+    z_min_scaled, z_max_scaled = z_min * resolution[0], z_max * resolution[0]
+    y_min_scaled, y_max_scaled = y_min * resolution[1], y_max * resolution[1]
+    x_min_scaled, x_max_scaled = x_min * resolution[2], x_max * resolution[2]
+
+    # z, y and x coordinates of the root mesh (extent of the whole object)
+    mesh_points = root_mesh.points
+    z_coords, y_coords, x_coords = (
+        mesh_points[:, 0],
+        mesh_points[:, 1],
+        mesh_points[:, 2],
+    )
+
+    # minimum and maximum coordinates of the root mesh
+    z_min_mesh, z_max_mesh = np.min(z_coords), np.max(z_coords)
+    y_min_mesh, y_max_mesh = np.min(y_coords), np.max(y_coords)
+    x_min_mesh, x_max_mesh = np.min(x_coords), np.max(x_coords)
+
+    # checking if root mesh and image are on the same scale
+    _assert_close(z_min_mesh, z_min_scaled, resolution[0])
+    _assert_close(z_max_mesh, z_max_scaled, resolution[0])
+    _assert_close(y_min_mesh, y_min_scaled, resolution[1])
+    _assert_close(y_max_mesh, y_max_scaled, resolution[1])
+    _assert_close(x_min_mesh, x_min_scaled, resolution[2])
+    _assert_close(x_max_mesh, x_max_scaled, resolution[2])
+
+    return True
+
+
+def open_for_visual_check():
+    # implement visual checks later
+    pass
+
+
+def validate_checksum():
+    # implement later
+    pass
+
+
+def check_additional_references():
+    # check additional references are different, but have same dimensions
+    pass
+
+
+def validate_atlas(atlas_name, version, all_validation_functions):
+    """Validates the latest version of a given atlas"""
+
+    print(atlas_name, version)
+    BrainGlobeAtlas(atlas_name)
+    updated = get_atlases_lastversions()[atlas_name]["updated"]
+    if not updated:
+        update_atlas(atlas_name)
+
+    validation_function_parameters = [
+        # validate_atlas_files(atlas_path: Path)
+        (Path(get_brainglobe_dir() / f"{atlas_name}_v{version}"),),
+        # validate_mesh_matches_image_extents(atlas: BrainGlobeAtlas)
+        (BrainGlobeAtlas(atlas_name),),
+        # open_for_visual_check()
+        (),
+        # validate_checksum()
+        (),
+        # check_additional_references()
+        (),
+    ]
+
+    # list to store the errors of the failed validations
+    failed_validations = []
+    successful_validations = []
+
+    for i, validation_function in enumerate(all_validation_functions):
+        try:
+            validation_function(*validation_function_parameters[i])
+            successful_validations.append((atlas_name, validation_function))
+        except AssertionError as error:
+            failed_validations.append((atlas_name, validation_function, error))
+
+    return successful_validations, failed_validations
+
+
+if __name__ == "__main__":
+    # list to store the validation functions
+    all_validation_functions = [
+        validate_atlas_files,
+        validate_mesh_matches_image_extents,
+        open_for_visual_check,
+        validate_checksum,
+        check_additional_references,
+    ]
+
+    valid_atlases = []
+    invalid_atlases = []
+    for atlas_name, version in get_all_atlases_lastversions().items():
+        successful_validations, failed_validations = validate_atlas(
+            atlas_name, version, all_validation_functions
+        )
+        for item in successful_validations:
+            valid_atlases.append(item)
+        for item in failed_validations:
+            invalid_atlases.append(item)
+
+    print("Summary")
+    print("### Valid atlases ###")
+    print(valid_atlases)
+    print("### Invalid atlases ###")
+    print(invalid_atlases)

pyproject.toml

@@ -45,6 +45,19 @@ allenmouse = [
     "allensdk",
 ]
 
+dev = [
+  "pytest",
+  "pytest-cov",
+  "pytest-mock",
+  "coverage",
+  "tox",
+  "black",
+  "mypy",
+  "pre-commit",
+  "ruff",
+  "setuptools_scm",
+]
+
 [build-system]
 requires = [
     "setuptools>=45",
@@ -59,12 +72,6 @@ include-package-data = true
 [tool.setuptools.packages.find]
 include = ["bg_atlasgen*"]
 
-[tool.pytest.ini_options]
-addopts = "--cov=bg_atlasgen"
-filterwarnings = [
-    "error",
-]
-
 
 [tool.black]
 target-version = ['py38', 'py39', 'py310', 'py311']

tests/test_unit/test_validation.py

@@ -0,0 +1,54 @@
+from pathlib import Path
+
+import numpy as np
+import pytest
+from bg_atlasapi import BrainGlobeAtlas
+from bg_atlasapi.config import get_brainglobe_dir
+
+from bg_atlasgen.validate_atlases import (
+    _assert_close,
+    validate_atlas_files,
+    validate_mesh_matches_image_extents,
+)
+
+
+def test_validate_mesh_matches_image_extents():
+    atlas = BrainGlobeAtlas("allen_mouse_100um")
+    assert validate_mesh_matches_image_extents(atlas)
+
+
+def test_validate_mesh_matches_image_extents_negative(mocker):
+    atlas = BrainGlobeAtlas("allen_mouse_100um")
+    flipped_annotation_image = np.transpose(atlas.annotation)
+    mocker.patch(
+        "bg_atlasapi.BrainGlobeAtlas.annotation",
+        new_callable=mocker.PropertyMock,
+        return_value=flipped_annotation_image,
+    )
+    with pytest.raises(
+        AssertionError, match="differ by more than 10 times pixel size"
+    ):
+        validate_mesh_matches_image_extents(atlas)
+
+
+def test_valid_atlas_files():
+    _ = BrainGlobeAtlas("allen_mouse_100um")
+    atlas_path = Path(get_brainglobe_dir()) / "allen_mouse_100um_v1.2"
+    assert validate_atlas_files(atlas_path)
+
+
+def test_invalid_atlas_path():
+    atlas_path = Path.home()
+    with pytest.raises(AssertionError, match="Expected file not found"):
+        validate_atlas_files(atlas_path)
+
+
+def test_assert_close():
+    assert _assert_close(99.5, 8, 10)
+
+
+def test_assert_close_negative():
+    with pytest.raises(
+        AssertionError, match="differ by more than 10 times pixel size"
+    ):
+        _assert_close(99.5, 30, 2)

tox.ini

@@ -1,9 +1,8 @@
 [tox]
-envlist = py{38,39,310,311}
+envlist = py{39,310,311}
 
 [gh-actions]
 python =
-    3.8: py38
     3.9: py39
     3.10: py310
     3.11: py311
@@ -12,4 +11,4 @@ python =
 extras =
     dev
 commands =
-    python -c "import bg_atlasgen"
+    pytest -v --color=yes --cov=bg_atlasgen --cov-report=xml