diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 68ef58741..8242f5996 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -6,7 +6,8 @@ variables: PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip RUN_COVERAGE: no PYTEST_ADDOPTS: --color=yes --junitxml=test-data/test-results.xml - PRERELEASE_DEPENDENCIES: no + DEPENDENCIES_VERSION: "latest" # |"pre-release" | "minimum-version" + TEST_TYPE: "standard" # | "coverage" jobs: - job: PyTest @@ -17,11 +18,17 @@ jobs: Python3.11: python.version: "3.11" RUN_COVERAGE: yes + TEST_TYPE: "coverage" Python3.9: python.version: "3.9" PreRelease: python.version: "3.11" - PRERELEASE_DEPENDENCIES: yes + DEPENDENCIES_VERSION: "pre-release" + TEST_TYPE: "strict-warning" + minimum_versions: + python.version: "3.9" + DEPENDENCIES_VERSION: "minimum" + TEST_TYPE: "coverage" steps: - task: UsePythonVersion@0 inputs: @@ -41,13 +48,20 @@ jobs: python -m pip install --upgrade pip wheel pip install .[dev,test] displayName: "Install dependencies" - condition: eq(variables['PRERELEASE_DEPENDENCIES'], 'no') + condition: eq(variables['DEPENDENCIES_VERSION'], 'latest') + + - script: | + python -m pip install pip wheel tomli packaging pytest-cov + pip install `python3 ci/scripts/min-deps.py pyproject.toml --extra dev test` + pip install --no-deps . + displayName: "Install minimum dependencies" + condition: eq(variables['DEPENDENCIES_VERSION'], 'minimum') - script: | python -m pip install --pre --upgrade pip wheel pip install --pre .[dev,test] displayName: "Install dependencies release candidates" - condition: eq(variables['PRERELEASE_DEPENDENCIES'], 'yes') + condition: eq(variables['DEPENDENCIES_VERSION'], 'pre-release') - script: | pip list @@ -56,23 +70,23 @@ jobs: - script: | pytest displayName: "PyTest" - condition: and(eq(variables['RUN_COVERAGE'], 'no'), eq(variables['PRERELEASE_DEPENDENCIES'], 'no')) + condition: eq(variables['TEST_TYPE'], 'standard') - script: | pytest --cov --cov-report=xml --cov-context=test displayName: "PyTest (coverage)" - condition: and(eq(variables['RUN_COVERAGE'], 'yes'), eq(variables['PRERELEASE_DEPENDENCIES'], 'no')) + condition: eq(variables['TEST_TYPE'], 'coverage') - script: | pytest --strict-warnings displayName: "PyTest (treat warnings as errors)" - condition: and(eq(variables['RUN_COVERAGE'], 'no'), eq(variables['PRERELEASE_DEPENDENCIES'], 'yes')) + condition: eq(variables['TEST_TYPE'], 'strict-warning') - task: PublishCodeCoverageResults@1 inputs: codeCoverageTool: Cobertura summaryFileLocation: "test-data/coverage.xml" - condition: eq(variables['RUN_COVERAGE'], 'yes') + condition: eq(variables['TEST_TYPE'], 'coverage') - task: PublishTestResults@2 condition: succeededOrFailed() @@ -83,7 +97,7 @@ jobs: - script: bash <(curl -s https://codecov.io/bash) displayName: "Upload to codecov.io" - condition: eq(variables['RUN_COVERAGE'], 'yes') + condition: eq(variables['TEST_TYPE'], 'coverage') - job: CheckBuild pool: diff --git a/anndata/experimental/merge.py b/anndata/experimental/merge.py index 95b0b215f..a3ffc1555 100644 --- a/anndata/experimental/merge.py +++ b/anndata/experimental/merge.py @@ -523,7 +523,7 @@ def concat_on_disk( >>> adata = ad.read_h5ad('merged.h5ad', backed=True) >>> adata.X CSRDataset: backend hdf5, shape (490, 15585), data_dtype float32 - >>> adata.obs['dataset'].value_counts() + >>> adata.obs['dataset'].value_counts() # doctest: +SKIP dataset fetal 344 b_cells 146 diff --git a/anndata/tests/test_concatenate.py b/anndata/tests/test_concatenate.py index c0d670cd0..e24431436 100644 --- a/anndata/tests/test_concatenate.py +++ b/anndata/tests/test_concatenate.py @@ -13,6 +13,7 @@ import pytest from boltons.iterutils import default_exit, remap, research from numpy import ma +from packaging.version import Version from scipy import sparse from anndata import AnnData, Raw, concat @@ -1350,7 +1351,7 @@ def test_concat_size_0_dim(axis, join_type, merge_strategy, shape): FutureWarning, match=r"The behavior of DataFrame concatenation with empty or all-NA entries is deprecated", ) - if shape[axis] == 0 + if shape[axis] == 0 and Version(pd.__version__) >= Version("2.1") else nullcontext() ) with ctx_concat_empty: diff --git a/anndata/tests/test_io_warnings.py b/anndata/tests/test_io_warnings.py index f6ed3e128..29ab2d963 100644 --- a/anndata/tests/test_io_warnings.py +++ b/anndata/tests/test_io_warnings.py @@ -5,7 +5,9 @@ from importlib.util import find_spec from pathlib import Path +import h5py import pytest +from packaging.version import Version import anndata as ad from anndata.tests.helpers import gen_adata @@ -43,6 +45,13 @@ def test_old_format_warning_not_thrown(tmp_path): with warnings.catch_warnings(record=True) as record: warnings.simplefilter("always", ad.OldFormatWarning) + if Version(h5py.__version__) < Version("3.2"): + # https://github.com/h5py/h5py/issues/1808 + warnings.filterwarnings( + "ignore", + r"Passing None into shape arguments as an alias for \(\) is deprecated\.", + category=DeprecationWarning, + ) ad.read_h5ad(pth) diff --git a/ci/scripts/min-deps.py b/ci/scripts/min-deps.py new file mode 100755 index 000000000..b3f393ea5 --- /dev/null +++ b/ci/scripts/min-deps.py @@ -0,0 +1,99 @@ +#!python3 +from __future__ import annotations + +import argparse +import sys +from collections import deque +from pathlib import Path +from typing import TYPE_CHECKING + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + +from packaging.requirements import Requirement +from packaging.version import Version + +if TYPE_CHECKING: + from collections.abc import Generator, Iterable + + +def min_dep(req: Requirement) -> Requirement: + """ + Given a requirement, return the minimum version specifier. + + Example + ------- + + >>> min_dep(Requirement("numpy>=1.0")) + "numpy==1.0" + """ + req_name = req.name + if req.extras: + req_name = f"{req_name}[{','.join(req.extras)}]" + + if not req.specifier: + return Requirement(req_name) + + min_version = Version("0.0.0.a1") + for spec in req.specifier: + if spec.operator in [">", ">=", "~="]: + min_version = max(min_version, Version(spec.version)) + elif spec.operator == "==": + min_version = Version(spec.version) + + return Requirement(f"{req_name}=={min_version}.*") + + +def extract_min_deps( + dependencies: Iterable[Requirement], *, pyproject +) -> Generator[Requirement, None, None]: + dependencies = deque(dependencies) # We'll be mutating this + project_name = pyproject["project"]["name"] + + while len(dependencies) > 0: + req = dependencies.pop() + + # If we are referring to other optional dependency lists, resolve them + if req.name == project_name: + assert req.extras, f"Project included itself as dependency, without specifying extras: {req}" + for extra in req.extras: + extra_deps = pyproject["project"]["optional-dependencies"][extra] + dependencies += map(Requirement, extra_deps) + else: + yield min_dep(req) + + +def main(): + parser = argparse.ArgumentParser( + prog="min-deps", + description="""Parse a pyproject.toml file and output a list of minimum dependencies. + + Output is directly passable to `pip install`.""", + usage="pip install `python min-deps.py pyproject.toml`", + ) + parser.add_argument( + "path", type=Path, help="pyproject.toml to parse minimum dependencies from" + ) + parser.add_argument( + "--extras", type=str, nargs="*", default=(), help="extras to install" + ) + + args = parser.parse_args() + + pyproject = tomllib.loads(args.path.read_text()) + + project_name = pyproject["project"]["name"] + deps = [ + *map(Requirement, pyproject["project"]["dependencies"]), + *(Requirement(f"{project_name}[{extra}]") for extra in args.extras), + ] + + min_deps = extract_min_deps(deps, pyproject=pyproject) + + print(" ".join(map(str, min_deps))) + + +if __name__ == "__main__": + main() diff --git a/docs/release-notes/0.10.5.md b/docs/release-notes/0.10.5.md index 8ffb759a6..b3a5f2142 100644 --- a/docs/release-notes/0.10.5.md +++ b/docs/release-notes/0.10.5.md @@ -17,3 +17,8 @@ * `BaseCompressedSparseDataset`'s `indptr` is cached {pr}`1266` {user}`ilan-gold` * Improved performance when indexing backed sparse matrices with boolean masks along their major axis {pr}`1233` {user}`ilan-gold` + +```{rubric} Development +``` + +* `anndata`'s CI now tests against minimum versions of it's dependencies. As a result, several dependencies had their minimum required version bumped. See diff for details {pr}`1314` {user}`ivirshup` diff --git a/docs/release-notes/0.10.6.md b/docs/release-notes/0.10.6.md index 76495d20c..097c1bc05 100644 --- a/docs/release-notes/0.10.6.md +++ b/docs/release-notes/0.10.6.md @@ -12,3 +12,8 @@ ```{rubric} Performance ``` + +```{rubric} Dev Process +``` + +* AnnData now tests against the minimum versions it's compatible with {pr}`1314` {user}`ivirshup` diff --git a/pyproject.toml b/pyproject.toml index 663fb21b8..41c91155b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,16 +36,15 @@ classifiers = [ "Topic :: Scientific/Engineering :: Visualization", ] dependencies = [ - # pandas <1.1.1 has pandas/issues/35446 + # pandas <1.4 has pandas/issues/35446 # pandas 2.1.0rc0 has pandas/issues/54622 - # pandas 2.1.2 has pandas/issues/52927 - "pandas >=1.1.1, !=2.1.0rc0, !=2.1.2", - "numpy>=1.16.5", # required by pandas 1.x - "scipy>1.4", - "h5py>=3", + "pandas >=1.4, !=2.1.0rc0, !=2.1.2", + "numpy>=1.23", + "scipy>1.8", + "h5py>=3.1", "exceptiongroup; python_version<'3.11'", "natsort", - "packaging>=20", + "packaging>=20.0", "array_api_compat", ] dynamic = ["version"] @@ -81,7 +80,7 @@ doc = [ ] test = [ "loompy>=3.0.5", - "pytest >=6.0", + "pytest>=7.3", "pytest-cov>=2.10", "zarr", "matplotlib", @@ -91,7 +90,7 @@ test = [ "boltons", "scanpy", "httpx", # For data downloading - "dask[array,distributed]", + "dask[array,distributed]>=2022.09.2", "awkward>=2.3", "pyarrow", "pytest_memray",