Update to ruff; fail on all warnings

.github/workflows/pytest.yml

@@ -1,4 +1,4 @@
-name: pytest
+name: Test
 
 on:
   push:

.pre-commit-config.yaml

@@ -4,29 +4,18 @@ repos:
     hooks:
       - id: absolufy-imports
         name: absolufy-imports
-  - repo: https://github.com/pycqa/isort
-    rev: 5.13.2
-    hooks:
-      - id: isort
-        language_version: python3
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.15.0
-    hooks:
-      - id: pyupgrade
-        args:
-          - --py38-plus
   - repo: https://github.com/psf/black
     rev: 23.12.1
     hooks:
       - id: black
         language_version: python3
         args:
           - --target-version=py38
-  - repo: https://github.com/pycqa/flake8
-    rev: 7.0.0
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.1.14
     hooks:
-      - id: flake8
-        language_version: python3
+      - id: ruff
+        args: ["--fix", "--show-fixes"]
   - repo: https://github.com/pre-commit/mirrors-mypy
     rev: v1.8.0
     hooks:

HOW_TO_RELEASE

@@ -13,32 +13,31 @@ Time required: about an hour.
       git commit -a -m 'Release vX.Y.Z'
  5. Tag the release:
       git tag -a vX.Y.Z -m 'vX.Y.Z'
- 6. Build source for pypi:
-      python setup.py sdist
- 7. Use twine to register and upload the release on pypi. Be careful, you can't
-    take this back!
-      twine upload dist/recursive_diff-X.Y.Z*
-    You will need to be listed as a package owner at
-    https://pypi.python.org/pypi/recursive_diff for this to work.
- 8. Push your changes to main:
+ 6. Push your changes to main:
       git push origin main
       git push origin --tags
- 9. Update the stable branch (used by ReadTheDocs) and switch back to main:
+ 7. Update the stable branch (used by ReadTheDocs) and switch back to main:
       git checkout stable
       git rebase main
       git push origin stable
       git checkout main
     It's OK to force push to 'stable' if necessary.
     We also update the stable branch with `git cherrypick` for documentation
     only fixes that apply the current released version.
-10. Add a section for the next release (v.X.(Y+1)) to doc/whats-new.rst.
-11. Commit your changes and push to main again:
+ 8. Add a section for the next release (v.X.(Y+1)) to doc/whats-new.rst.
+ 9. Commit your changes and push to main again:
       git commit -a -m 'Revert to dev version'
       git push origin main
     You're done pushing to main!
-12. Issue the release on GitHub. Open https://github.com/crusaderky/recursive_diff/releases;
+10. Issue the release on GitHub. Open https://github.com/crusaderky/recursive_diff/releases;
     the new release should have automatically appeared. Otherwise, click on
     "Draft a new release" and paste in the latest from whats-new.rst.
+11. Download the .tar.gz package for the release
+12. Use twine to register and upload the release on pypi. Be careful, you can't
+    take this back!
+      twine upload recursive_diff-*.tar.gz
+    You will need to be listed as a package owner at
+    https://pypi.python.org/pypi/recursive_diff for this to work.
 13. Update the docs. Login to https://readthedocs.org/projects/recursive_diff/versions/
     and switch your new release tag (at the bottom) from "Inactive" to "Active".
     It should now build automatically.

ci/requirements-no_optionals.yml

@@ -7,3 +7,4 @@ dependencies:
   - packaging
 
   - xarray
+  - pyarrow  # Only needed to suppress warnings; remove when pandas 3.0 is released

pyproject.toml

@@ -0,0 +1,127 @@
+[project]
+name = "recursive_diff"
+authors = [{name = "Guido Imperiale", email = "[email protected]"}]
+license = {text = "Apache"}
+description = "Recursively compare two Python data structures"
+classifiers = [
+    "Development Status :: 5 - Production/Stable",
+    "License :: OSI Approved :: Apache Software License",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering",
+    "Operating System :: OS Independent",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.8",
+    "Programming Language :: Python :: 3.9",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+]
+requires-python = ">=3.8"
+dependencies = [
+    "numpy >= 1.16",
+    "pandas >= 0.25",
+    "xarray >= 0.12",
+]
+dynamic = ["version"]
+
+[project.urls]
+Homepage = "https://github.com/crusaderky/recursive_diff"
+
+[project.scripts]
+ncdiff = "recursive_diff.ncdiff:main"
+
+[tool.setuptools]
+packages = ["recursive_diff"]
+zip-safe = false # https://mypy.readthedocs.io/en/latest/installed_packages.html
+include-package-data = true
+
+[tool.setuptools_scm]
+# Use hardcoded version when .git has been removed and this is not a package created
+# by sdist. This is the case e.g. of a remote deployment with PyCharm.
+fallback_version = "9999"
+
+[tool.setuptools.package-data]
+recursive_diff = ["py.typed"]
+
+[build-system]
+requires = [
+    "setuptools>=66",
+    "setuptools_scm[toml]",
+]
+build-backend = "setuptools.build_meta"
+
+[tool.pytest.ini_options]
+addopts = "--strict-markers --strict-config -v -r sxfE --color=yes"
+xfail_strict = true
+python_files = ["test_*.py"]
+testpaths = ["recursive_diff/tests"]
+filterwarnings = [
+    "error",
+    # Raised internally by pandas
+    'ignore:datetime.datetime.utcfromtimestamp\(\) is deprecated:DeprecationWarning',
+    # numpy <1.26 only
+    'ignore:elementwise comparison failed:DeprecationWarning',
+    'ignore:elementwise comparison failed:FutureWarning',
+    'ignore:invalid value encountered in cast:RuntimeWarning',
+    # Deprecations in proper_unstack
+    # FIXME https://github.com/crusaderky/xarray_extras/issues/33
+    'ignore:the `pandas.MultiIndex` object.* will no longer be implicitly promoted:FutureWarning',
+    'ignore:updating coordinate .* with a PandasMultiIndex:FutureWarning',
+    'ignore:Updating MultiIndexed coordinate .* would corrupt indices:FutureWarning',
+]
+
+[tool.coverage.report]
+show_missing = true
+exclude_lines = [
+    "pragma: nocover",
+    "pragma: no cover",
+    "TYPE_CHECKING",
+    "except ImportError",
+    "@overload",
+    '@(abc\.)?abstractmethod',
+]
+
+[tool.ruff]
+builtins = ["ellipsis"]
+exclude = [".eggs"]
+target-version = "py38"
+
+[tool.ruff.lint]
+ignore = [
+  "E402",  # module level import not at top of file
+  "SIM108",  # use ternary operator instead of if-else block
+  "N999",  # Invalid module name: 'TEMPLATE'  TODO remove this line
+]
+select = [
+  "F",    # Pyflakes
+  "B",    # flake8-bugbear
+  "C4",   # flake8-comprehensions
+  "ISC",  # flake8-implicit-str-concat
+  "SIM",  # flake8-simplify
+  "E",    # Pycodestyle
+  "W",    # Pycodestyle
+  "I",    # isort
+  "N",    # pep8-naming
+  "UP",   # Pyupgrade
+  "RUF",  # unused-noqa
+  "EXE001",  #  Shebang is present but file is not executable
+]
+
+[tool.ruff.lint.isort]
+known-first-party = ["TEMPLATE"]
+
+[tool.mypy]
+allow_incomplete_defs = false
+allow_untyped_decorators = false
+allow_untyped_defs = false
+ignore_missing_imports = true
+no_implicit_optional = true
+show_error_codes = true
+warn_redundant_casts = true
+warn_unused_ignores = true
+warn_unreachable = true
+
+[[tool.mypy.overrides]]
+module = ["*.tests.*"]
+allow_untyped_defs = true

recursive_diff/proper_unstack.py

@@ -1,27 +1,19 @@
-"""Copy-pasted from xarray-extras
+"""Utilities for stacking/unstacking dimensions
+
+Copy-pasted from xarray-extras
 """
 from __future__ import annotations
 
 from collections.abc import Hashable
-from typing import overload
+from typing import TypeVar
 
 import pandas
 import xarray
 
+T = TypeVar("T", xarray.DataArray, xarray.Dataset)
 
-@overload
-def proper_unstack(array: xarray.DataArray, dim: Hashable) -> xarray.DataArray:
-    ...
-
-
-@overload
-def proper_unstack(array: xarray.Dataset, dim: Hashable) -> xarray.Dataset:
-    ...
 
-
-def proper_unstack(
-    array: xarray.DataArray | xarray.Dataset, dim: Hashable
-) -> xarray.DataArray | xarray.Dataset:
+def proper_unstack(array: T, dim: Hashable) -> T:
     """Work around an issue in xarray that causes the data to be sorted
     alphabetically by label on unstack():
 
@@ -37,24 +29,25 @@ def proper_unstack(
     :param Hashable dim:
         Name of existing dimension to unstack
     :returns:
-        xarray.DataArray / xarray.Dataset with unstacked dimension
+        xarray.DataArray or xarray.Dataset with unstacked dimension
     """
     # Regenerate Pandas multi-index to be ordered by first appearance
     mindex = array.coords[dim].to_pandas().index
 
     levels = []
-    labels = []
-    for levels_i, labels_i in zip(mindex.levels, mindex.codes):
-        level_map: dict[str, int] = {}
+    codes = []
+
+    for levels_i, codes_i in zip(mindex.levels, mindex.codes):
+        level_map: dict[Hashable, int] = {}
 
-        for label in labels_i:
-            if label not in level_map:
-                level_map[label] = len(level_map)
+        for code in codes_i:
+            if code not in level_map:
+                level_map[code] = len(level_map)
 
-        levels.append([levels_i[k] for k in level_map.keys()])
-        labels.append([level_map[k] for k in labels_i])
+        levels.append([levels_i[k] for k in level_map])
+        codes.append([level_map[k] for k in codes_i])
 
-    mindex = pandas.MultiIndex(levels, labels, names=mindex.names)
+    mindex = pandas.MultiIndex(levels, codes, names=mindex.names)
     array = array.copy()
     array.coords[dim] = mindex
 

recursive_diff/recursive_diff.py

@@ -213,7 +213,7 @@ def diff(msg: str, print_path: list[object] = path) -> str:
                 rel_tol=rel_tol,
                 abs_tol=abs_tol,
                 brief_dims=brief_dims,
-                path=path + [i],
+                path=[*path, i],
                 suppress_type_diffs=suppress_type_diffs,
                 join=join,
             )
@@ -269,15 +269,15 @@ def diff(msg: str, print_path: list[object] = path) -> str:
                 rel_tol=rel_tol,
                 abs_tol=abs_tol,
                 brief_dims=brief_dims,
-                path=path + [key],
+                path=[*path, key],
                 suppress_type_diffs=suppress_type_diffs,
                 join=join,
             )
 
     elif are_instances(lhs, rhs, bool):
         if lhs != rhs:
             yield diff(f"{lhs} != {rhs}")
-    elif are_instances(lhs, rhs, str):
+    elif are_instances(lhs, rhs, str):  # noqa: SIM114
         if lhs != rhs:
             yield diff(f"{lhs_repr} != {rhs_repr}")
     elif are_instances(lhs, rhs, bytes):
@@ -390,7 +390,7 @@ def diff(msg: str, print_path: list[object] = path) -> str:
                 # Convert the diff count to plain dict with the original coords
                 diffs = _dataarray_to_dict(diffs)
                 for k, count in sorted(diffs.items()):
-                    yield diff(f"{count} differences", print_path=path + [k])
+                    yield diff(f"{count} differences", print_path=[*path, k])
 
             elif "__stacked__" not in lhs.dims:
                 # N>0 original dimensions, all of which are in brief_dims

recursive_diff/recursive_eq.py

@@ -13,7 +13,7 @@ def recursive_eq(
     """
     diffs_iter = recursive_diff(lhs, rhs, rel_tol=rel_tol, abs_tol=abs_tol)
     i = -1
-    for i, diff in enumerate(diffs_iter):
+    for i, diff in enumerate(diffs_iter):  # noqa: B007
         print(diff)
     i += 1
     assert i == 0, f"{i} differences found"

recursive_diff/tests/test_ncdiff.py

@@ -111,7 +111,7 @@ def test_singlefile(tmpdir, capsys, argv, out):
     a.to_netcdf(f"{tmpdir}/a.nc")
     b.to_netcdf(f"{tmpdir}/b.nc")
 
-    exit_code = main(argv + [f"{tmpdir}/a.nc", f"{tmpdir}/b.nc"])
+    exit_code = main([*argv, f"{tmpdir}/a.nc", f"{tmpdir}/b.nc"])
     assert exit_code == 1
     assert_stdout(capsys, out)
 
@@ -137,7 +137,7 @@ def test_singlefile(tmpdir, capsys, argv, out):
         (
             ["-r", "lhs", "rhs", "-m", "**/a.nc"],
             "[" + os.path.join("subdir", "a.nc") + "][data_vars][d1][x=10]: 1 != -9 "
-            "(abs: -1.0e+01, rel: -1.0e+01)\n"  # noqa
+            "(abs: -1.0e+01, rel: -1.0e+01)\n"
             "Found 1 differences\n",
         ),
     ],

recursive_diff/tests/test_recursive_diff.py

@@ -354,6 +354,9 @@ def test_numpy_string_slice(x, y):
     check(b, c)
 
 
+@pytest.mark.filterwarnings(
+    "ignore:Converting non-nanosecond precision datetime:UserWarning"
+)
 def test_numpy_dates():
     a = pd.to_datetime(["2000-01-01", "2000-01-02", "2000-01-03", "NaT"]).values.astype(
         "<M8[D]"
@@ -455,17 +458,17 @@ def test_pandas_rangeindex():
     check(
         pd.RangeIndex(0, 4, 1),
         pd.RangeIndex(1, 4, 1),
-        "RangeIndex(start=0, stop=4, step=1) != " "RangeIndex(start=1, stop=4, step=1)",
+        "RangeIndex(start=0, stop=4, step=1) != RangeIndex(start=1, stop=4, step=1)",
     )
     check(
         pd.RangeIndex(0, 4, 2),
         pd.RangeIndex(0, 5, 2),
-        "RangeIndex(start=0, stop=4, step=2) != " "RangeIndex(start=0, stop=5, step=2)",
+        "RangeIndex(start=0, stop=4, step=2) != RangeIndex(start=0, stop=5, step=2)",
     )
     check(
         pd.RangeIndex(0, 4, 2),
         pd.RangeIndex(0, 4, 3),
-        "RangeIndex(start=0, stop=4, step=2) != " "RangeIndex(start=0, stop=4, step=3)",
+        "RangeIndex(start=0, stop=4, step=2) != RangeIndex(start=0, stop=4, step=3)",
     )
     check(
         pd.RangeIndex(4, name="foo"),
@@ -824,7 +827,7 @@ def test_custom_classes():
     check(
         Rectangle(4, 4),
         Square(4),
-        "Cannot compare objects: Rectangle(4, 4), Square(4)",  # noqa: E501
+        "Cannot compare objects: Rectangle(4, 4), Square(4)",
         "object type differs: Rectangle != Square",
     )
 
@@ -839,7 +842,7 @@ def test_custom_classes():
 @requires_dask
 @pytest.mark.parametrize(
     "chunk_lhs,chunk_rhs",
-    [(None, None), (None, -1), (None, 2), (((1, 2),), ((2, 1),))],
+    [(None, None), (None, -1), (None, 2), ({"x": (1, 2)}, {"x": (2, 1)})],
 )
 def test_dask(chunk_lhs, chunk_rhs):
     lhs = xarray.DataArray(["a", "b", "c"], dims=["x"])