Merge branch 'main' into fix-release-notes

.azure-pipelines.yml

@@ -6,22 +6,29 @@ variables:
   PIP_CACHE_DIR: $(Pipeline.Workspace)/.pip
   RUN_COVERAGE: no
   PYTEST_ADDOPTS: --color=yes --junitxml=test-data/test-results.xml
-  PRERELEASE_DEPENDENCIES: no
+  DEPENDENCIES_VERSION: "latest" # |"pre-release" | "minimum-version"
+  TEST_TYPE: "standard" # | "coverage"
 
 jobs:
   - job: PyTest
     pool:
       vmImage: "ubuntu-22.04"
     strategy:
       matrix:
-        Python3.11:
-          python.version: "3.11"
+        Python3.12:
+          python.version: "3.12"
           RUN_COVERAGE: yes
+          TEST_TYPE: "coverage"
         Python3.9:
           python.version: "3.9"
         PreRelease:
-          python.version: "3.11"
-          PRERELEASE_DEPENDENCIES: yes
+          python.version: "3.12"
+          DEPENDENCIES_VERSION: "pre-release"
+          TEST_TYPE: "strict-warning"
+        minimum_versions:
+          python.version: "3.9"
+          DEPENDENCIES_VERSION: "minimum"
+          TEST_TYPE: "coverage"
     steps:
       - task: UsePythonVersion@0
         inputs:
@@ -41,13 +48,20 @@ jobs:
           python -m pip install --upgrade pip wheel
           pip install .[dev,test]
         displayName: "Install dependencies"
-        condition: eq(variables['PRERELEASE_DEPENDENCIES'], 'no')
+        condition: eq(variables['DEPENDENCIES_VERSION'], 'latest')
+
+      - script: |
+          python -m pip install pip wheel tomli packaging pytest-cov
+          pip install `python3 ci/scripts/min-deps.py pyproject.toml --extra dev test`
+          pip install --no-deps .
+        displayName: "Install minimum dependencies"
+        condition: eq(variables['DEPENDENCIES_VERSION'], 'minimum')
 
       - script: |
           python -m pip install --pre --upgrade pip wheel
           pip install --pre .[dev,test]
         displayName: "Install dependencies release candidates"
-        condition: eq(variables['PRERELEASE_DEPENDENCIES'], 'yes')
+        condition: eq(variables['DEPENDENCIES_VERSION'], 'pre-release')
 
       - script: |
           pip list
@@ -56,23 +70,23 @@ jobs:
       - script: |
           pytest
         displayName: "PyTest"
-        condition: and(eq(variables['RUN_COVERAGE'], 'no'), eq(variables['PRERELEASE_DEPENDENCIES'], 'no'))
+        condition: eq(variables['TEST_TYPE'], 'standard')
 
       - script: |
           pytest --cov --cov-report=xml --cov-context=test
         displayName: "PyTest (coverage)"
-        condition: and(eq(variables['RUN_COVERAGE'], 'yes'), eq(variables['PRERELEASE_DEPENDENCIES'], 'no'))
+        condition: eq(variables['TEST_TYPE'], 'coverage')
 
       - script: |
           pytest --strict-warnings
         displayName: "PyTest (treat warnings as errors)"
-        condition: and(eq(variables['RUN_COVERAGE'], 'no'), eq(variables['PRERELEASE_DEPENDENCIES'], 'yes'))
+        condition: eq(variables['TEST_TYPE'], 'strict-warning')
 
       - task: PublishCodeCoverageResults@1
         inputs:
           codeCoverageTool: Cobertura
           summaryFileLocation: "test-data/coverage.xml"
-        condition: eq(variables['RUN_COVERAGE'], 'yes')
+        condition: eq(variables['TEST_TYPE'], 'coverage')
 
       - task: PublishTestResults@2
         condition: succeededOrFailed()
@@ -83,16 +97,16 @@ jobs:
 
       - script: bash <(curl -s https://codecov.io/bash)
         displayName: "Upload to codecov.io"
-        condition: eq(variables['RUN_COVERAGE'], 'yes')
+        condition: eq(variables['TEST_TYPE'], 'coverage')
 
   - job: CheckBuild
     pool:
       vmImage: "ubuntu-22.04"
     steps:
       - task: UsePythonVersion@0
         inputs:
-          versionSpec: "3.11"
-        displayName: "Use Python 3.11"
+          versionSpec: "3.12"
+        displayName: "Use Python 3.12"
 
       - script: |
           python -m pip install --upgrade pip

.codecov.yml

@@ -8,7 +8,6 @@ coverage:
       default:
         # Require 1% coverage, i.e., always succeed
         target: 1
-    patch: false
     changes: false
 
 comment:

.github/workflows/benchmark.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python: ["3.11"]
+        python: ["3.12"]
         os: [ubuntu-latest]
 
     env:

.github/workflows/test-gpu.yml

@@ -35,9 +35,14 @@ jobs:
     name: GPU Tests
     needs: check
     runs-on: "cirun-aws-gpu--${{ github.run_id }}"
+    # Setting a timeout of 30 minutes, as the AWS costs money
+    # At time of writing, a typical run takes about 5 minutes
+    timeout-minutes: 30
+
     defaults:
       run:
         shell: bash -el {0}
+
     steps:
       - uses: actions/checkout@v3
         with:
@@ -49,23 +54,18 @@ jobs:
       - uses: mamba-org/setup-micromamba@v1
         with:
           micromamba-version: "1.3.1-0"
-          environment-name: anndata-gpu-ci
-          create-args: >-
-            python=3.11
-            cupy
-            numba
-            pytest
-            pytest-cov
-            pytest-xdist
+          environment-file: ci/gpu_ci.yml
           init-shell: >-
             bash
           generate-run-shell: false
 
       - name: Install AnnData
         run: pip install .[dev,test,gpu]
 
-      - name: Mamba list
-        run: micromamba list
+      - name: Env list
+        run: |
+          micromamba list
+          pip list
 
       - name: Run test
         run: pytest -m gpu --cov --cov-report=xml --cov-context=test -n 4

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.1.14
+    rev: v0.3.2
     hooks:
       - id: ruff
         types_or: [python, pyi, jupyter]
@@ -13,7 +13,6 @@ repos:
       - id: prettier
         exclude_types:
           - markdown
-        language_version: 21.5.0
   - repo: https://github.com/pre-commit/pre-commit-hooks
     rev: v4.5.0
     hooks:

.readthedocs.yml

@@ -2,7 +2,7 @@ version: 2
 build:
   os: ubuntu-20.04
   tools:
-    python: "3.11"
+    python: "3.12"
 sphinx:
   configuration: docs/conf.py
   fail_on_warning: true # do not change or you will be fired

anndata/__init__.py

@@ -1,4 +1,5 @@
 """Annotated multivariate observation data."""
+
 from __future__ import annotations
 
 try:  # See https://github.com/maresb/hatch-vcs-footgun-example
@@ -34,6 +35,7 @@
     read_umi_tools,
     read_zarr,
 )
+from ._settings import settings
 from ._warnings import (
     ExperimentalFeatureWarning,
     ImplicitModificationWarning,
@@ -44,6 +46,9 @@
 # Experimental needs to be imported last
 from . import experimental  # isort: skip
 
+# We use these in tests by attribute access
+from . import _io, logging  # noqa: F401 isort: skip
+
 
 def read(*args, **kwargs):
     import warnings
@@ -75,4 +80,5 @@ def read(*args, **kwargs):
     "ImplicitModificationWarning",
     "ExperimentalFeatureWarning",
     "experimental",
+    "settings",
 ]

anndata/_core/aligned_df.py

@@ -0,0 +1,104 @@
+from __future__ import annotations
+
+import warnings
+from functools import singledispatch
+from typing import TYPE_CHECKING, Any, Literal
+
+import pandas as pd
+from pandas.api.types import is_string_dtype
+
+from .._warnings import ImplicitModificationWarning
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable, Mapping
+
+
+@singledispatch
+def _gen_dataframe(
+    anno: Mapping[str, Any],
+    index_names: Iterable[str],
+    *,
+    source: Literal["X", "shape"],
+    attr: Literal["obs", "var"],
+    length: int | None = None,
+) -> pd.DataFrame:
+    if anno is None or len(anno) == 0:
+        anno = {}
+
+    def mk_index(l: int) -> pd.Index:
+        return pd.RangeIndex(0, l, name=None).astype(str)
+
+    for index_name in index_names:
+        if index_name not in anno:
+            continue
+        df = pd.DataFrame(
+            anno,
+            index=anno[index_name],
+            columns=[k for k in anno.keys() if k != index_name],
+        )
+        break
+    else:
+        df = pd.DataFrame(
+            anno,
+            index=None if length is None else mk_index(length),
+            columns=None if len(anno) else [],
+        )
+
+    if length is None:
+        df.index = mk_index(len(df))
+    elif length != len(df):
+        raise _mk_df_error(source, attr, length, len(df))
+    return df
+
+
+@_gen_dataframe.register(pd.DataFrame)
+def _gen_dataframe_df(
+    anno: pd.DataFrame,
+    index_names: Iterable[str],
+    *,
+    source: Literal["X", "shape"],
+    attr: Literal["obs", "var"],
+    length: int | None = None,
+):
+    if length is not None and length != len(anno):
+        raise _mk_df_error(source, attr, length, len(anno))
+    anno = anno.copy(deep=False)
+    if not is_string_dtype(anno.index):
+        warnings.warn("Transforming to str index.", ImplicitModificationWarning)
+        anno.index = anno.index.astype(str)
+    if not len(anno.columns):
+        anno.columns = anno.columns.astype(str)
+    return anno
+
+
+@_gen_dataframe.register(pd.Series)
+@_gen_dataframe.register(pd.Index)
+def _gen_dataframe_1d(
+    anno: pd.Series | pd.Index,
+    index_names: Iterable[str],
+    *,
+    source: Literal["X", "shape"],
+    attr: Literal["obs", "var"],
+    length: int | None = None,
+):
+    raise ValueError(f"Cannot convert {type(anno)} to {attr} DataFrame")
+
+
+def _mk_df_error(
+    source: Literal["X", "shape"],
+    attr: Literal["obs", "var"],
+    expected: int,
+    actual: int,
+):
+    if source == "X":
+        what = "row" if attr == "obs" else "column"
+        msg = (
+            f"Observations annot. `{attr}` must have as many rows as `X` has {what}s "
+            f"({expected}), but has {actual} rows."
+        )
+    else:
+        msg = (
+            f"`shape` is inconsistent with `{attr}` "
+            "({actual} {what}s instead of {expected})"
+        )
+    return ValueError(msg)