From 2fb3b074c79e34e4dd1d8b3cbecde2181b7b485e Mon Sep 17 00:00:00 2001 From: crusaderky Date: Mon, 12 Feb 2024 17:49:22 +0000 Subject: [PATCH] Update to ruff; fail on all warnings --- .github/workflows/pytest.yml | 2 +- .pre-commit-config.yaml | 19 +-- HOW_TO_RELEASE | 23 ++-- ci/requirements-no_optionals.yml | 1 + pyproject.toml | 127 ++++++++++++++++++++ recursive_diff/proper_unstack.py | 41 +++---- recursive_diff/recursive_diff.py | 8 +- recursive_diff/recursive_eq.py | 2 +- recursive_diff/tests/test_ncdiff.py | 4 +- recursive_diff/tests/test_recursive_diff.py | 13 +- setup.cfg | 96 --------------- setup.py | 5 +- 12 files changed, 177 insertions(+), 164 deletions(-) create mode 100644 pyproject.toml delete mode 100644 setup.cfg diff --git a/.github/workflows/pytest.yml b/.github/workflows/pytest.yml index 18e4f1e..010ce37 100644 --- a/.github/workflows/pytest.yml +++ b/.github/workflows/pytest.yml @@ -1,4 +1,4 @@ -name: pytest +name: Test on: push: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 657f0d6..234935b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,17 +4,6 @@ repos: hooks: - id: absolufy-imports name: absolufy-imports - - repo: https://github.com/pycqa/isort - rev: 5.13.2 - hooks: - - id: isort - language_version: python3 - - repo: https://github.com/asottile/pyupgrade - rev: v3.15.0 - hooks: - - id: pyupgrade - args: - - --py38-plus - repo: https://github.com/psf/black rev: 23.12.1 hooks: @@ -22,11 +11,11 @@ repos: language_version: python3 args: - --target-version=py38 - - repo: https://github.com/pycqa/flake8 - rev: 7.0.0 + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.1.14 hooks: - - id: flake8 - language_version: python3 + - id: ruff + args: ["--fix", "--show-fixes"] - repo: https://github.com/pre-commit/mirrors-mypy rev: v1.8.0 hooks: diff --git a/HOW_TO_RELEASE b/HOW_TO_RELEASE index ff3b9e1..a6e451e 100644 --- a/HOW_TO_RELEASE +++ b/HOW_TO_RELEASE @@ -13,17 +13,10 @@ Time required: about an hour. git commit -a -m 'Release vX.Y.Z' 5. Tag the release: git tag -a vX.Y.Z -m 'vX.Y.Z' - 6. Build source for pypi: - python setup.py sdist - 7. Use twine to register and upload the release on pypi. Be careful, you can't - take this back! - twine upload dist/recursive_diff-X.Y.Z* - You will need to be listed as a package owner at - https://pypi.python.org/pypi/recursive_diff for this to work. - 8. Push your changes to main: + 6. Push your changes to main: git push origin main git push origin --tags - 9. Update the stable branch (used by ReadTheDocs) and switch back to main: + 7. Update the stable branch (used by ReadTheDocs) and switch back to main: git checkout stable git rebase main git push origin stable @@ -31,14 +24,20 @@ Time required: about an hour. It's OK to force push to 'stable' if necessary. We also update the stable branch with `git cherrypick` for documentation only fixes that apply the current released version. -10. Add a section for the next release (v.X.(Y+1)) to doc/whats-new.rst. -11. Commit your changes and push to main again: + 8. Add a section for the next release (v.X.(Y+1)) to doc/whats-new.rst. + 9. Commit your changes and push to main again: git commit -a -m 'Revert to dev version' git push origin main You're done pushing to main! -12. Issue the release on GitHub. Open https://github.com/crusaderky/recursive_diff/releases; +10. Issue the release on GitHub. Open https://github.com/crusaderky/recursive_diff/releases; the new release should have automatically appeared. Otherwise, click on "Draft a new release" and paste in the latest from whats-new.rst. +11. Download the .tar.gz package for the release +12. Use twine to register and upload the release on pypi. Be careful, you can't + take this back! + twine upload recursive_diff-*.tar.gz + You will need to be listed as a package owner at + https://pypi.python.org/pypi/recursive_diff for this to work. 13. Update the docs. Login to https://readthedocs.org/projects/recursive_diff/versions/ and switch your new release tag (at the bottom) from "Inactive" to "Active". It should now build automatically. diff --git a/ci/requirements-no_optionals.yml b/ci/requirements-no_optionals.yml index 5ab0710..c003c0c 100644 --- a/ci/requirements-no_optionals.yml +++ b/ci/requirements-no_optionals.yml @@ -7,3 +7,4 @@ dependencies: - packaging - xarray + - pyarrow # Only needed to suppress warnings; remove when pandas 3.0 is released diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..a2605ad --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,127 @@ +[project] +name = "recursive_diff" +authors = [{name = "Guido Imperiale", email = "crusaderky@gmail.com"}] +license = {text = "Apache"} +description = "Recursively compare two Python data structures" +classifiers = [ + "Development Status :: 5 - Production/Stable", + "License :: OSI Approved :: Apache Software License", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", +] +requires-python = ">=3.8" +dependencies = [ + "numpy >= 1.16", + "pandas >= 0.25", + "xarray >= 0.12", +] +dynamic = ["version"] + +[project.urls] +Homepage = "https://github.com/crusaderky/recursive_diff" + +[project.scripts] +ncdiff = "recursive_diff.ncdiff:main" + +[tool.setuptools] +packages = ["recursive_diff"] +zip-safe = false # https://mypy.readthedocs.io/en/latest/installed_packages.html +include-package-data = true + +[tool.setuptools_scm] +# Use hardcoded version when .git has been removed and this is not a package created +# by sdist. This is the case e.g. of a remote deployment with PyCharm. +fallback_version = "9999" + +[tool.setuptools.package-data] +recursive_diff = ["py.typed"] + +[build-system] +requires = [ + "setuptools>=66", + "setuptools_scm[toml]", +] +build-backend = "setuptools.build_meta" + +[tool.pytest.ini_options] +addopts = "--strict-markers --strict-config -v -r sxfE --color=yes" +xfail_strict = true +python_files = ["test_*.py"] +testpaths = ["recursive_diff/tests"] +filterwarnings = [ + "error", + # Raised internally by pandas + 'ignore:datetime.datetime.utcfromtimestamp\(\) is deprecated:DeprecationWarning', + # numpy <1.26 only + 'ignore:elementwise comparison failed:DeprecationWarning', + 'ignore:elementwise comparison failed:FutureWarning', + 'ignore:invalid value encountered in cast:RuntimeWarning', + # Deprecations in proper_unstack + # FIXME https://github.com/crusaderky/xarray_extras/issues/33 + 'ignore:the `pandas.MultiIndex` object.* will no longer be implicitly promoted:FutureWarning', + 'ignore:updating coordinate .* with a PandasMultiIndex:FutureWarning', + 'ignore:Updating MultiIndexed coordinate .* would corrupt indices:FutureWarning', +] + +[tool.coverage.report] +show_missing = true +exclude_lines = [ + "pragma: nocover", + "pragma: no cover", + "TYPE_CHECKING", + "except ImportError", + "@overload", + '@(abc\.)?abstractmethod', +] + +[tool.ruff] +builtins = ["ellipsis"] +exclude = [".eggs"] +target-version = "py38" + +[tool.ruff.lint] +ignore = [ + "E402", # module level import not at top of file + "SIM108", # use ternary operator instead of if-else block + "N999", # Invalid module name: 'TEMPLATE' TODO remove this line +] +select = [ + "F", # Pyflakes + "B", # flake8-bugbear + "C4", # flake8-comprehensions + "ISC", # flake8-implicit-str-concat + "SIM", # flake8-simplify + "E", # Pycodestyle + "W", # Pycodestyle + "I", # isort + "N", # pep8-naming + "UP", # Pyupgrade + "RUF", # unused-noqa + "EXE001", # Shebang is present but file is not executable +] + +[tool.ruff.lint.isort] +known-first-party = ["TEMPLATE"] + +[tool.mypy] +allow_incomplete_defs = false +allow_untyped_decorators = false +allow_untyped_defs = false +ignore_missing_imports = true +no_implicit_optional = true +show_error_codes = true +warn_redundant_casts = true +warn_unused_ignores = true +warn_unreachable = true + +[[tool.mypy.overrides]] +module = ["*.tests.*"] +allow_untyped_defs = true diff --git a/recursive_diff/proper_unstack.py b/recursive_diff/proper_unstack.py index 5c8ad02..dc2478b 100644 --- a/recursive_diff/proper_unstack.py +++ b/recursive_diff/proper_unstack.py @@ -1,27 +1,19 @@ -"""Copy-pasted from xarray-extras +"""Utilities for stacking/unstacking dimensions + +Copy-pasted from xarray-extras """ from __future__ import annotations from collections.abc import Hashable -from typing import overload +from typing import TypeVar import pandas import xarray +T = TypeVar("T", xarray.DataArray, xarray.Dataset) -@overload -def proper_unstack(array: xarray.DataArray, dim: Hashable) -> xarray.DataArray: - ... - - -@overload -def proper_unstack(array: xarray.Dataset, dim: Hashable) -> xarray.Dataset: - ... - -def proper_unstack( - array: xarray.DataArray | xarray.Dataset, dim: Hashable -) -> xarray.DataArray | xarray.Dataset: +def proper_unstack(array: T, dim: Hashable) -> T: """Work around an issue in xarray that causes the data to be sorted alphabetically by label on unstack(): @@ -37,24 +29,25 @@ def proper_unstack( :param Hashable dim: Name of existing dimension to unstack :returns: - xarray.DataArray / xarray.Dataset with unstacked dimension + xarray.DataArray or xarray.Dataset with unstacked dimension """ # Regenerate Pandas multi-index to be ordered by first appearance mindex = array.coords[dim].to_pandas().index levels = [] - labels = [] - for levels_i, labels_i in zip(mindex.levels, mindex.codes): - level_map: dict[str, int] = {} + codes = [] + + for levels_i, codes_i in zip(mindex.levels, mindex.codes): + level_map: dict[Hashable, int] = {} - for label in labels_i: - if label not in level_map: - level_map[label] = len(level_map) + for code in codes_i: + if code not in level_map: + level_map[code] = len(level_map) - levels.append([levels_i[k] for k in level_map.keys()]) - labels.append([level_map[k] for k in labels_i]) + levels.append([levels_i[k] for k in level_map]) + codes.append([level_map[k] for k in codes_i]) - mindex = pandas.MultiIndex(levels, labels, names=mindex.names) + mindex = pandas.MultiIndex(levels, codes, names=mindex.names) array = array.copy() array.coords[dim] = mindex diff --git a/recursive_diff/recursive_diff.py b/recursive_diff/recursive_diff.py index 06dcfbf..59a0b53 100755 --- a/recursive_diff/recursive_diff.py +++ b/recursive_diff/recursive_diff.py @@ -213,7 +213,7 @@ def diff(msg: str, print_path: list[object] = path) -> str: rel_tol=rel_tol, abs_tol=abs_tol, brief_dims=brief_dims, - path=path + [i], + path=[*path, i], suppress_type_diffs=suppress_type_diffs, join=join, ) @@ -269,7 +269,7 @@ def diff(msg: str, print_path: list[object] = path) -> str: rel_tol=rel_tol, abs_tol=abs_tol, brief_dims=brief_dims, - path=path + [key], + path=[*path, key], suppress_type_diffs=suppress_type_diffs, join=join, ) @@ -277,7 +277,7 @@ def diff(msg: str, print_path: list[object] = path) -> str: elif are_instances(lhs, rhs, bool): if lhs != rhs: yield diff(f"{lhs} != {rhs}") - elif are_instances(lhs, rhs, str): + elif are_instances(lhs, rhs, str): # noqa: SIM114 if lhs != rhs: yield diff(f"{lhs_repr} != {rhs_repr}") elif are_instances(lhs, rhs, bytes): @@ -390,7 +390,7 @@ def diff(msg: str, print_path: list[object] = path) -> str: # Convert the diff count to plain dict with the original coords diffs = _dataarray_to_dict(diffs) for k, count in sorted(diffs.items()): - yield diff(f"{count} differences", print_path=path + [k]) + yield diff(f"{count} differences", print_path=[*path, k]) elif "__stacked__" not in lhs.dims: # N>0 original dimensions, all of which are in brief_dims diff --git a/recursive_diff/recursive_eq.py b/recursive_diff/recursive_eq.py index d0ef0e0..2311707 100644 --- a/recursive_diff/recursive_eq.py +++ b/recursive_diff/recursive_eq.py @@ -13,7 +13,7 @@ def recursive_eq( """ diffs_iter = recursive_diff(lhs, rhs, rel_tol=rel_tol, abs_tol=abs_tol) i = -1 - for i, diff in enumerate(diffs_iter): + for i, diff in enumerate(diffs_iter): # noqa: B007 print(diff) i += 1 assert i == 0, f"{i} differences found" diff --git a/recursive_diff/tests/test_ncdiff.py b/recursive_diff/tests/test_ncdiff.py index 7170d7f..b926453 100644 --- a/recursive_diff/tests/test_ncdiff.py +++ b/recursive_diff/tests/test_ncdiff.py @@ -111,7 +111,7 @@ def test_singlefile(tmpdir, capsys, argv, out): a.to_netcdf(f"{tmpdir}/a.nc") b.to_netcdf(f"{tmpdir}/b.nc") - exit_code = main(argv + [f"{tmpdir}/a.nc", f"{tmpdir}/b.nc"]) + exit_code = main([*argv, f"{tmpdir}/a.nc", f"{tmpdir}/b.nc"]) assert exit_code == 1 assert_stdout(capsys, out) @@ -137,7 +137,7 @@ def test_singlefile(tmpdir, capsys, argv, out): ( ["-r", "lhs", "rhs", "-m", "**/a.nc"], "[" + os.path.join("subdir", "a.nc") + "][data_vars][d1][x=10]: 1 != -9 " - "(abs: -1.0e+01, rel: -1.0e+01)\n" # noqa + "(abs: -1.0e+01, rel: -1.0e+01)\n" "Found 1 differences\n", ), ], diff --git a/recursive_diff/tests/test_recursive_diff.py b/recursive_diff/tests/test_recursive_diff.py index a53bd08..ad94a2b 100644 --- a/recursive_diff/tests/test_recursive_diff.py +++ b/recursive_diff/tests/test_recursive_diff.py @@ -354,6 +354,9 @@ def test_numpy_string_slice(x, y): check(b, c) +@pytest.mark.filterwarnings( + "ignore:Converting non-nanosecond precision datetime:UserWarning" +) def test_numpy_dates(): a = pd.to_datetime(["2000-01-01", "2000-01-02", "2000-01-03", "NaT"]).values.astype( "=3.8 -install_requires = - numpy >= 1.16 - pandas >= 0.25 - xarray >= 0.12 -setup_requires = setuptools_scm - -[options.package_data] -recursive_diff = - py.typed - -[options.entry_points] -console_scripts = - ncdiff = recursive_diff.ncdiff:main - -[bdist_wheel] -universal = 1 - -[wheel] -universal = 1 - -[tool:pytest] -addopts = --strict -python_files = test_*.py -testpaths = recursive_diff/tests - -[coverage:report] -show_missing = true -exclude_lines = - pragma: nocover - pragma: no cover - TYPE_CHECKING - except ImportError - @overload - @(abc\.)?abstractmethod - -[flake8] -# https://github.com/python/black#line-length -max-line-length = 88 -# E203: PEP8-compliant slice operators -# https://github.com/python/black#slices -# W503: Allow for breaks before binary operator (Knuth's convention) - see -# https://www.python.org/dev/peps/pep-0008/#should-a-line-break-before-or-after-a-binary-operator -ignore = E203, W503 -exclude = - .eggs - doc/ - -[isort] -default_section = THIRDPARTY -known_first_party = recursive_diff -multi_line_output = 3 -include_trailing_comma = True -force_grid_wrap = 0 -use_parentheses = True -line_length = 88 - -[mypy] -allow_incomplete_defs = false -allow_untyped_decorators = false -allow_untyped_defs = false -ignore_missing_imports = true -no_implicit_optional = true -show_error_codes = true -warn_redundant_casts = true -warn_unused_ignores = true -warn_unreachable = true - -[mypy-*.tests.*] -allow_untyped_defs = true diff --git a/setup.py b/setup.py index 281a796..d5d43d7 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,3 @@ -#!/usr/bin/env python from setuptools import setup -# Use hardcoded version when .git has been removed and this is not a package created by -# sdist. This is the case e.g. of a remote deployment with PyCharm. -setup(use_scm_version={"fallback_version": "999"}) +setup(use_scm_version=True)