Debug failing regression test

.github/workflows/test_and_deploy.yml

@@ -3,10 +3,11 @@ name: tests
 on:
   push:
     branches:
-      - '*'
+      - 'main'
     tags:
-      - '*'
+      - 'v*'
   pull_request:
+  workflow_dispatch:
 
 jobs:
   linting:

.gitignore

@@ -117,3 +117,4 @@ venv.bak/
 .idea/
 .vs/
 *.~lock.*
+.vscode/

pyproject.toml

@@ -1,36 +1,36 @@
 [project]
 name = "brainglobe-utils"
-authors = [{name = "Adam Tyson", email= "[email protected]"}]
+authors = [{ name = "Adam Tyson", email = "[email protected]" }]
 description = "Shared general purpose tools for the BrainGlobe project"
 readme = "README.md"
 requires-python = ">=3.9.0"
 dynamic = ["version"]
 
 dependencies = [
-    "natsort",
-    "pandas",
-    "psutil",
-    "configobj",
-    "tqdm",
-    "PyYAML",
-    "scipy",
-    "numpy",
-    "slurmio",
-    "tifffile",
-    "imio",
-    "bg-atlasapi",
+  "natsort",
+  "pandas",
+  "psutil",
+  "configobj",
+  "tqdm",
+  "PyYAML",
+  "scipy",
+  "numpy",
+  "slurmio",
+  "tifffile",
+  "imio",
+  "bg-atlasapi",
 ]
 
-license = {text = "MIT"}
+license = { text = "MIT" }
 
 classifiers = [
-    "Development Status :: 2 - Pre-Alpha",
-    "Programming Language :: Python",
-    "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.9",
-    "Programming Language :: Python :: 3.10",
-    "Programming Language :: Python :: 3.11",
-    "Operating System :: OS Independent",
+  "Development Status :: 2 - Pre-Alpha",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Operating System :: OS Independent",
 ]
 
 [project.urls]
@@ -40,11 +40,7 @@ source_code = "https://github.com/brainglobe/brainglobe-utils"
 user_support = "https://github.com/brainglobe/brainglobe-utils/issues"
 
 [project.optional-dependencies]
-napari = [
-  "napari>=0.4.18",
-  "qtpy",
-  "superqt"
-]
+napari = ["napari>=0.4.18", "qtpy", "superqt"]
 
 dev = [
   "qtpy",
@@ -60,16 +56,12 @@ dev = [
   "setuptools_scm",
   "pyqt5",
   "superqt",
-  "scikit-image"
+  "scikit-image",
 ]
 
 
 [build-system]
-requires = [
-    "setuptools>=45",
-    "wheel",
-    "setuptools_scm[toml]>=6.2",
-]
+requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools]
@@ -104,7 +96,7 @@ ignore = [
 
 [tool.ruff]
 line-length = 79
-exclude = ["__init__.py","build",".eggs"]
+exclude = ["__init__.py", "build", ".eggs"]
 select = ["I", "E", "F"]
 fix = true
 
@@ -115,3 +107,28 @@ ignore_errors = true
 module = ["imlib.cells.*"]
 ignore_errors = false
 strict = true
+
+[tool.tox]
+legacy_tox_ini = """
+[tox]
+envlist = py{39,310,311}
+isolated_build = True
+
+[gh-actions]
+python =
+    3.9: py39
+    3.10: py310
+    3.11: py311
+
+[testenv]
+extras =
+    dev
+commands =
+    pytest -v --color=yes --cov=brainglobe_utils --cov-report=xml
+passenv =
+    CI
+    GITHUB_ACTIONS
+    DISPLAY
+    XAUTHORITY
+    PYVISTA_OFF_SCREEN
+"""

tests/tests/test_brainmapper/test_analysis.py

@@ -1,6 +1,6 @@
-import filecmp
 from pathlib import Path
 
+import pandas as pd
 import pytest
 
 from brainglobe_utils.brainmapper.analysis import (
@@ -37,9 +37,17 @@ def structures_with_points():
 
 
 def test_get_region_totals(tmp_path, points, structures_with_points):
-    """Regression test for count_points_per_brain_region for
+    """
+    Regression test for count_points_per_brain_region for
     pandas 1.5.3 -> 2.1.3+.
-    pd.Dataframe.append was deprecated and removed in this time."""
+    pd.DataFrame.append was deprecated and removed in this time.
+
+    2024-01-31: Newer versions of pandas are writing the
+    DataFrame rows in a different order. As such, filecmp is no longer
+    sufficient to check that we are still counting the number of cells
+    correctly - we will need to load the data back in and do a pandas
+    comparison.
+    """
     OUTPUT_DATA_LOC = (
         Path(__file__).parent.parent.parent / "data" / "brainmapper"
     )
@@ -54,4 +62,21 @@ def test_get_region_totals(tmp_path, points, structures_with_points):
         points, structures_with_points, volumes_path, output_path
     )
     assert output_path.exists()
-    assert filecmp.cmp(output_path, expected_output)
+
+    # Read data back in, and sort rows by the structures for comparison.
+    # NOTE: Column 'structure name' should exist, otherwise we've failed
+    # and the test will flag the error appropriately
+    expected_df = (
+        pd.read_csv(expected_output)
+        .sort_values("structure_name")
+        .reset_index(drop=True)
+    )
+    produced_df = (
+        pd.read_csv(output_path)
+        .sort_values("structure_name")
+        .reset_index(drop=True)
+    )
+    assert expected_df.equals(
+        produced_df
+    ), "Produced DataFrame no longer matches per-region "
+    "count from expected DataFrame"