From 501d54563b5d8614a79192e73cb1e9cb66ea75b2 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Mon, 13 Jan 2025 15:20:44 -0800 Subject: [PATCH] Change how optional deps are set up, tested --- .github/PULL_REQUEST_TEMPLATE/release.md | 11 ++-- .github/workflows/check_sphinx_links.yml | 5 +- .github/workflows/deploy_release.yml | 4 +- .github/workflows/run_all_tests.yml | 80 +++++++++++------------ .github/workflows/run_coverage.yml | 22 +++---- .github/workflows/run_hdmf_zarr_tests.yml | 2 +- .github/workflows/run_pynwb_tests.yml | 2 +- .github/workflows/run_tests.yml | 27 ++++---- .gitignore | 1 + .readthedocs.yaml | 9 +-- docs/source/install_developers.rst | 7 +- docs/source/software_process.rst | 42 ++++++------ docs/source/update_requirements.rst | 78 ---------------------- environment-ros3.yml | 9 ++- pyproject.toml | 38 +++++++++-- requirements-dev.txt | 14 ---- requirements-doc.txt | 6 -- requirements-opt.txt | 6 -- requirements.txt | 8 --- tox.ini | 48 +++++++------- 20 files changed, 158 insertions(+), 261 deletions(-) delete mode 100644 docs/source/update_requirements.rst delete mode 100644 requirements-dev.txt delete mode 100644 requirements-doc.txt delete mode 100644 requirements-opt.txt delete mode 100644 requirements.txt diff --git a/.github/PULL_REQUEST_TEMPLATE/release.md b/.github/PULL_REQUEST_TEMPLATE/release.md index 86a7ad57d..882f4313e 100644 --- a/.github/PULL_REQUEST_TEMPLATE/release.md +++ b/.github/PULL_REQUEST_TEMPLATE/release.md @@ -2,11 +2,8 @@ Prepare for release of HDMF [version] ### Before merging: - [ ] Make sure all PRs to be included in this release have been merged to `dev`. -- [ ] Major and minor releases: Update package versions in `requirements.txt`, `requirements-dev.txt`, - `requirements-doc.txt`, `requirements-opt.txt`, and `environment-ros3.yml` to the latest versions, - and update dependency ranges in `pyproject.toml` and minimums in `requirements-min.txt` as needed. - Run `pip install pur && pur -r requirements-dev.txt -r requirements.txt -r requirements-opt.txt` - and manually update `environment-ros3.yml`. +- [ ] Major and minor releases: Update dependency ranges in `pyproject.toml` and minimums in + `requirements-min.txt` as needed. - [ ] Check legal file dates and information in `Legal.txt`, `license.txt`, `README.rst`, `docs/source/conf.py`, and any other locations as needed - [ ] Update `pyproject.toml` as needed @@ -34,5 +31,5 @@ Prepare for release of HDMF [version] 4. Either monitor [conda-forge/hdmf-feedstock](https://github.com/conda-forge/hdmf-feedstock) for the regro-cf-autotick-bot bot to create a PR updating the version of HDMF to the latest PyPI release, usually within 24 hours of release, or manually create a PR updating `recipe/meta.yaml` with the latest version number - and SHA256 retrieved from PyPI > HDMF > Download Files > View hashes for the `.tar.gz` file. Re-render and update - dependencies as needed. + and SHA256 retrieved from PyPI > HDMF > Download Files > View hashes for the `.tar.gz` file. Re-render and + update the dependencies as needed. diff --git a/.github/workflows/check_sphinx_links.yml b/.github/workflows/check_sphinx_links.yml index 15fc61e30..0ce64b011 100644 --- a/.github/workflows/check_sphinx_links.yml +++ b/.github/workflows/check_sphinx_links.yml @@ -21,13 +21,12 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' # TODO update to 3.12 when optional reqs (e.g., oaklib) support 3.12 + python-version: '3.13' - name: Install Sphinx dependencies and package run: | python -m pip install --upgrade pip - python -m pip install -r requirements-doc.txt -r requirements-opt.txt - python -m pip install . + python -m pip install ".[test,docs,full]" - name: Check Sphinx internal and external links run: sphinx-build -W -b linkcheck ./docs/source ./test_build diff --git a/.github/workflows/deploy_release.yml b/.github/workflows/deploy_release.yml index 5861ab136..ab0db960a 100644 --- a/.github/workflows/deploy_release.yml +++ b/.github/workflows/deploy_release.yml @@ -18,7 +18,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - name: Install build dependencies run: | @@ -28,7 +28,7 @@ jobs: - name: Run tox tests run: | - tox -e py312-upgraded + tox -e py313-upgraded - name: Build wheel and source distribution run: | diff --git a/.github/workflows/run_all_tests.yml b/.github/workflows/run_all_tests.yml index 961500194..a3c5547cc 100644 --- a/.github/workflows/run_all_tests.yml +++ b/.github/workflows/run_all_tests.yml @@ -25,30 +25,27 @@ jobs: fail-fast: false matrix: include: - - { name: linux-python3.9-minimum , test-tox-env: pytest-py39-minimum , python-ver: "3.9" , os: ubuntu-latest } - - { name: linux-python3.10 , test-tox-env: pytest-py310-pinned , python-ver: "3.10", os: ubuntu-latest } - - { name: linux-python3.11 , test-tox-env: pytest-py311-pinned , python-ver: "3.11", os: ubuntu-latest } - - { name: linux-python3.11-optional , test-tox-env: pytest-py311-optional-pinned , python-ver: "3.11", os: ubuntu-latest } - - { name: linux-python3.12 , test-tox-env: pytest-py312-pinned , python-ver: "3.12", os: ubuntu-latest } - - { name: linux-python3.13 , test-tox-env: pytest-py313-pinned , python-ver: "3.13", os: ubuntu-latest } - - { name: linux-python3.13-upgraded , test-tox-env: pytest-py313-upgraded , python-ver: "3.13", os: ubuntu-latest } - - { name: linux-python3.13-prerelease , test-tox-env: pytest-py313-prerelease , python-ver: "3.13", os: ubuntu-latest } - - { name: windows-python3.9-minimum , test-tox-env: pytest-py39-minimum , python-ver: "3.9" , os: windows-latest } - - { name: windows-python3.10 , test-tox-env: pytest-py310-pinned , python-ver: "3.10", os: windows-latest } - - { name: windows-python3.11 , test-tox-env: pytest-py311-pinned , python-ver: "3.11", os: windows-latest } - - { name: windows-python3.11-optional , test-tox-env: pytest-py311-optional-pinned , python-ver: "3.11", os: windows-latest } - - { name: windows-python3.12 , test-tox-env: pytest-py312-pinned , python-ver: "3.12", os: windows-latest } - - { name: windows-python3.13 , test-tox-env: pytest-py313-pinned , python-ver: "3.13", os: windows-latest } - - { name: windows-python3.13-upgraded , test-tox-env: pytest-py313-upgraded , python-ver: "3.13", os: windows-latest } - - { name: windows-python3.13-prerelease , test-tox-env: pytest-py313-prerelease , python-ver: "3.13", os: windows-latest } - - { name: macos-python3.9-minimum , test-tox-env: pytest-py39-minimum , python-ver: "3.9" , os: macos-13 } - - { name: macos-python3.10 , test-tox-env: pytest-py310-pinned , python-ver: "3.10", os: macos-latest } - - { name: macos-python3.11 , test-tox-env: pytest-py311-pinned , python-ver: "3.11", os: macos-latest } - - { name: macos-python3.11-optional , test-tox-env: pytest-py311-optional-pinned , python-ver: "3.11", os: macos-latest } - - { name: macos-python3.12 , test-tox-env: pytest-py312-pinned , python-ver: "3.12", os: macos-latest } - - { name: macos-python3.13 , test-tox-env: pytest-py313-pinned , python-ver: "3.13", os: macos-latest } - - { name: macos-python3.13-upgraded , test-tox-env: pytest-py313-upgraded , python-ver: "3.13", os: macos-latest } - - { name: macos-python3.13-prerelease , test-tox-env: pytest-py313-prerelease , python-ver: "3.13", os: macos-latest } + - { name: linux-python3.9-minimum , test-tox-env: pytest-py39-minimum , python-ver: "3.9" , os: ubuntu-latest } + - { name: linux-python3.10-upgraded , test-tox-env: pytest-py310-upgraded , python-ver: "3.10", os: ubuntu-latest } + - { name: linux-python3.11-upgraded , test-tox-env: pytest-py311-upgraded , python-ver: "3.11", os: ubuntu-latest } + - { name: linux-python3.12-upgraded , test-tox-env: pytest-py312-upgraded , python-ver: "3.12", os: ubuntu-latest } + - { name: linux-python3.13-upgraded , test-tox-env: pytest-py313-upgraded , python-ver: "3.13", os: ubuntu-latest } + - { name: linux-python3.13-upgraded-optional , test-tox-env: pytest-py313-upgraded-optional , python-ver: "3.13", os: ubuntu-latest } + - { name: linux-python3.13-prerelease-optional , test-tox-env: pytest-py313-prerelease-optional , python-ver: "3.13", os: ubuntu-latest } + - { name: windows-python3.9-minimum , test-tox-env: pytest-py39-minimum , python-ver: "3.9" , os: windows-latest } + - { name: windows-python3.10-upgraded , test-tox-env: pytest-py310-upgraded , python-ver: "3.10", os: windows-latest } + - { name: windows-python3.11-upgraded , test-tox-env: pytest-py311-upgraded , python-ver: "3.11", os: windows-latest } + - { name: windows-python3.12-upgraded , test-tox-env: pytest-py312-upgraded , python-ver: "3.12", os: windows-latest } + - { name: windows-python3.13-upgraded , test-tox-env: pytest-py313-upgraded , python-ver: "3.13", os: windows-latest } + - { name: windows-python3.13-upgraded-optional , test-tox-env: pytest-py313-upgraded-optional , python-ver: "3.13", os: windows-latest } + - { name: windows-python3.13-prerelease-optional , test-tox-env: pytest-py313-prerelease-optional , python-ver: "3.13", os: windows-latest } + - { name: macos-python3.9-minimum , test-tox-env: pytest-py39-minimum , python-ver: "3.9" , os: macos-13 } + - { name: macos-python3.10-upgraded , test-tox-env: pytest-py310-upgraded , python-ver: "3.10", os: macos-latest } + - { name: macos-python3.11-upgraded , test-tox-env: pytest-py311-upgraded , python-ver: "3.11", os: macos-latest } + - { name: macos-python3.12-upgraded , test-tox-env: pytest-py312-upgraded , python-ver: "3.12", os: macos-latest } + - { name: macos-python3.13-upgraded , test-tox-env: pytest-py313-upgraded , python-ver: "3.13", os: macos-latest } + - { name: macos-python3.13-upgraded-optional , test-tox-env: pytest-py313-upgraded-optional , python-ver: "3.13", os: macos-latest } + - { name: macos-python3.13-prerelease-optional , test-tox-env: pytest-py313-prerelease-optional , python-ver: "3.13", os: macos-latest } steps: - name: Checkout repo with submodules uses: actions/checkout@v4 @@ -97,18 +94,15 @@ jobs: fail-fast: false matrix: include: - - { name: linux-gallery-python3.9-minimum , test-tox-env: gallery-py39-minimum , python-ver: "3.9" , os: ubuntu-latest } - - { name: linux-gallery-python3.11-optional , test-tox-env: gallery-py311-optional-pinned , python-ver: "3.11", os: ubuntu-latest } - - { name: linux-gallery-python3.13-upgraded , test-tox-env: gallery-py313-upgraded , python-ver: "3.13", os: ubuntu-latest } - - { name: linux-gallery-python3.13-prerelease , test-tox-env: gallery-py313-prerelease , python-ver: "3.13", os: ubuntu-latest } - - { name: windows-gallery-python3.9-minimum , test-tox-env: gallery-py39-minimum , python-ver: "3.9" , os: windows-latest } - - { name: windows-gallery-python3.11-optional , test-tox-env: gallery-py311-optional-pinned , python-ver: "3.11", os: windows-latest } - - { name: windows-gallery-python3.13-upgraded , test-tox-env: gallery-py313-upgraded , python-ver: "3.13", os: windows-latest } - - { name: windows-gallery-python3.13-prerelease, test-tox-env: gallery-py313-prerelease , python-ver: "3.13", os: windows-latest } - - { name: macos-gallery-python3.9-minimum , test-tox-env: gallery-py39-minimum , python-ver: "3.9" , os: macos-13 } - - { name: macos-gallery-python3.11-optional , test-tox-env: gallery-py311-optional-pinned , python-ver: "3.11", os: macos-latest } - - { name: macos-gallery-python3.13-upgraded , test-tox-env: gallery-py313-upgraded , python-ver: "3.13", os: macos-latest } - - { name: macos-gallery-python3.13-prerelease , test-tox-env: gallery-py313-prerelease , python-ver: "3.13", os: macos-latest } + - { name: linux-gallery-python3.9-minimum , test-tox-env: gallery-py39-minimum , python-ver: "3.9" , os: ubuntu-latest } + - { name: linux-gallery-python3.13-upgraded-optional , test-tox-env: gallery-py313-upgraded-optional , python-ver: "3.13", os: ubuntu-latest } + - { name: linux-gallery-python3.13-prerelease-optional , test-tox-env: gallery-py313-prerelease-optional , python-ver: "3.13", os: ubuntu-latest } + - { name: windows-gallery-python3.9-minimum , test-tox-env: gallery-py39-minimum , python-ver: "3.9" , os: windows-latest } + - { name: windows-gallery-python3.13-upgraded-optional , test-tox-env: gallery-py313-upgraded-optional , python-ver: "3.13", os: windows-latest } + - { name: windows-gallery-python3.13-prerelease-optional , test-tox-env: gallery-py313-prerelease-optional , python-ver: "3.13", os: windows-latest } + - { name: macos-gallery-python3.9-minimum , test-tox-env: gallery-py39-minimum , python-ver: "3.9" , os: macos-13 } + - { name: macos-gallery-python3.13-upgraded-optional , test-tox-env: gallery-py313-upgraded-optional , python-ver: "3.13", os: macos-latest } + - { name: macos-gallery-python3.13-prerelease-optional , test-tox-env: gallery-py313-prerelease-optional , python-ver: "3.13", os: macos-latest } steps: - name: Checkout repo with submodules uses: actions/checkout@v4 @@ -144,13 +138,13 @@ jobs: fail-fast: false matrix: include: - - { name: conda-linux-python3.9-minimum , test-tox-env: pytest-py39-minimum , python-ver: "3.9" , os: ubuntu-latest } - - { name: conda-linux-python3.10 , test-tox-env: pytest-py310-pinned , python-ver: "3.10", os: ubuntu-latest } - - { name: conda-linux-python3.11 , test-tox-env: pytest-py311-pinned , python-ver: "3.11", os: ubuntu-latest } - - { name: conda-linux-python3.11-optional , test-tox-env: pytest-py311-optional-pinned , python-ver: "3.11", os: ubuntu-latest } - - { name: conda-linux-python3.13 , test-tox-env: pytest-py313-pinned , python-ver: "3.13", os: ubuntu-latest } - - { name: conda-linux-python3.13-upgraded , test-tox-env: pytest-py313-upgraded , python-ver: "3.13", os: ubuntu-latest } - - { name: conda-linux-python3.13-prerelease , test-tox-env: pytest-py313-prerelease , python-ver: "3.13", os: ubuntu-latest } + - { name: conda-linux-python3.9-minimum , test-tox-env: pytest-py39-minimum , python-ver: "3.9" , os: ubuntu-latest } + - { name: conda-linux-python3.10-upgraded , test-tox-env: pytest-py310-upgraded , python-ver: "3.10", os: ubuntu-latest } + - { name: conda-linux-python3.11-upgraded , test-tox-env: pytest-py311-upgraded , python-ver: "3.11", os: ubuntu-latest } + - { name: conda-linux-python3.12-upgraded , test-tox-env: pytest-py312-upgraded , python-ver: "3.12", os: ubuntu-latest } + - { name: conda-linux-python3.13-upgraded , test-tox-env: pytest-py313-upgraded , python-ver: "3.13", os: ubuntu-latest } + - { name: conda-linux-python3.13-upgraded-optional , test-tox-env: pytest-py313-upgraded-optional , python-ver: "3.13", os: ubuntu-latest } + - { name: conda-linux-python3.13-prerelease-optional , test-tox-env: pytest-py313-prerelease-optional , python-ver: "3.13", os: ubuntu-latest } steps: - name: Checkout repo with submodules uses: actions/checkout@v4 diff --git a/.github/workflows/run_coverage.yml b/.github/workflows/run_coverage.yml index bea8df734..67399dabb 100644 --- a/.github/workflows/run_coverage.yml +++ b/.github/workflows/run_coverage.yml @@ -31,7 +31,7 @@ jobs: - { os: macos-latest , opt_req: false } env: # used by codecov-action OS: ${{ matrix.os }} - PYTHON: '3.11' # TODO update to 3.12 when optional reqs (e.g., oaklib) support 3.12 + PYTHON: '3.13' steps: - name: Checkout repo with submodules uses: actions/checkout@v4 @@ -44,24 +44,24 @@ jobs: with: python-version: ${{ env.PYTHON }} - - name: Install dependencies + - name: Upgrade pip run: | python -m pip install --upgrade pip - python -m pip install -r requirements-dev.txt -r requirements.txt - - - name: Install optional dependencies - if: ${{ matrix.opt_req }} - run: python -m pip install -r requirements-opt.txt - name: Install package - run: | - python -m pip install . - python -m pip list + if: ! ${{ matrix.opt_req }} + run: python -m pip install ".[test]" + + - name: Install package with optional dependencies + if: ${{ matrix.opt_req }} + run: python -m pip install ".[test,full]" - name: Run tests and generate coverage report run: | # coverage is configured in pyproject.toml - pytest --cov --cov-report=xml --cov-report=term # codecov uploader requires xml format + # codecov uploader requires xml format + python -m pip list + pytest --cov --cov-report=xml --cov-report=term - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 diff --git a/.github/workflows/run_hdmf_zarr_tests.yml b/.github/workflows/run_hdmf_zarr_tests.yml index 6eb5546ab..51a01977a 100644 --- a/.github/workflows/run_hdmf_zarr_tests.yml +++ b/.github/workflows/run_hdmf_zarr_tests.yml @@ -31,7 +31,7 @@ jobs: python -m pip list git clone https://github.com/hdmf-dev/hdmf-zarr.git cd hdmf-zarr - python -m pip install .[test] # this will install a different version of hdmf from the current one + python -m pip install ".[test]" # this will install a different version of hdmf from the current one cd .. python -m pip uninstall -y hdmf # uninstall the other version of hdmf python -m pip install . # reinstall current branch of hdmf diff --git a/.github/workflows/run_pynwb_tests.yml b/.github/workflows/run_pynwb_tests.yml index 1a714ed9f..a159380cd 100644 --- a/.github/workflows/run_pynwb_tests.yml +++ b/.github/workflows/run_pynwb_tests.yml @@ -21,7 +21,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - name: Update pip run: python -m pip install --upgrade pip diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index a3fa5a84c..7bd00eb61 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -23,13 +23,13 @@ jobs: matrix: include: # NOTE config below with "upload-wheels: true" specifies that wheels should be uploaded as an artifact - - { name: linux-python3.9-minimum , test-tox-env: pytest-py39-minimum , python-ver: "3.9" , os: ubuntu-latest } - - { name: linux-python3.13 , test-tox-env: pytest-py313-pinned , python-ver: "3.13", os: ubuntu-latest } - - { name: linux-python3.13-upgraded , test-tox-env: pytest-py313-upgraded , python-ver: "3.13", os: ubuntu-latest , upload-wheels: true } - - { name: windows-python3.9-minimum , test-tox-env: pytest-py39-minimum , python-ver: "3.9" , os: windows-latest } - - { name: windows-python3.13-upgraded , test-tox-env: pytest-py313-upgraded , python-ver: "3.13", os: windows-latest } - - { name: macos-python3.9-minimum , test-tox-env: pytest-py39-minimum , python-ver: "3.9" , os: macos-13 } - - { name: macos-python3.13-upgraded , test-tox-env: pytest-py313-upgraded , python-ver: "3.13", os: macos-latest } + - { name: linux-python3.9-minimum , test-tox-env: pytest-py39-minimum , python-ver: "3.9" , os: ubuntu-latest } + - { name: linux-python3.13-upgraded , test-tox-env: pytest-py313-upgraded , python-ver: "3.13", os: ubuntu-latest } + - { name: linux-python3.13-upgraded-optional , test-tox-env: pytest-py313-upgraded-optional , python-ver: "3.13", os: ubuntu-latest , upload-wheels: true } + - { name: windows-python3.9-minimum , test-tox-env: pytest-py39-minimum , python-ver: "3.9" , os: windows-latest } + - { name: windows-python3.13-upgraded-optional , test-tox-env: pytest-py313-upgraded-optional , python-ver: "3.13", os: windows-latest } + - { name: macos-python3.9-minimum , test-tox-env: pytest-py39-minimum , python-ver: "3.9" , os: macos-13 } + - { name: macos-python3.13-upgraded-optional , test-tox-env: pytest-py313-upgraded-optional , python-ver: "3.13", os: macos-latest } steps: - name: Checkout repo with submodules uses: actions/checkout@v4 @@ -85,10 +85,10 @@ jobs: fail-fast: false matrix: include: - - { name: linux-gallery-python3.9-minimum , test-tox-env: gallery-py39-minimum , python-ver: "3.9" , os: ubuntu-latest } - - { name: linux-gallery-python3.13-upgraded , test-tox-env: gallery-py313-upgraded , python-ver: "3.13", os: ubuntu-latest } - - { name: windows-gallery-python3.9-minimum , test-tox-env: gallery-py39-minimum , python-ver: "3.9" , os: windows-latest } - - { name: windows-gallery-python3.13-upgraded , test-tox-env: gallery-py313-upgraded , python-ver: "3.13", os: windows-latest } + - { name: linux-gallery-python3.9-minimum , test-tox-env: gallery-py39-minimum , python-ver: "3.9" , os: ubuntu-latest } + - { name: linux-gallery-python3.13-upgraded-optional , test-tox-env: gallery-py313-upgraded-optional , python-ver: "3.13", os: ubuntu-latest } + - { name: windows-gallery-python3.9-minimum , test-tox-env: gallery-py39-minimum , python-ver: "3.9" , os: windows-latest } + - { name: windows-gallery-python3.13-upgraded-optional , test-tox-env: gallery-py313-upgraded-optional , python-ver: "3.13", os: windows-latest } steps: - name: Checkout repo with submodules uses: actions/checkout@v4 @@ -124,8 +124,9 @@ jobs: fail-fast: false matrix: include: - - { name: conda-linux-python3.9-minimum , test-tox-env: pytest-py39-minimum , python-ver: "3.9" , os: ubuntu-latest } - - { name: conda-linux-python3.13-upgraded , test-tox-env: pytest-py313-upgraded , python-ver: "3.13", os: ubuntu-latest } + - { name: conda-linux-python3.9-minimum , test-tox-env: pytest-py39-minimum , python-ver: "3.9" , os: ubuntu-latest } + - { name: conda-linux-python3.13-upgraded , test-tox-env: pytest-py313-upgraded , python-ver: "3.13", os: ubuntu-latest } + - { name: conda-linux-python3.13-upgraded-optional , test-tox-env: pytest-py313-upgraded-optional , python-ver: "3.13", os: ubuntu-latest } steps: - name: Checkout repo with submodules uses: actions/checkout@v4 diff --git a/.gitignore b/.gitignore index d75abc985..e202b3526 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ # Auto-generated apidocs RST files /docs/source/gen_modules/ /docs/source/hdmf*.rst +/docs/source/sg_execution_times.rst /docs/gallery/*.hdf5 /docs/gallery/*.sqlite /docs/gallery/expanded_example_dynamic_term_set.yaml diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 19bcaee80..a200e362a 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -6,9 +6,9 @@ version: 2 build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: - python: '3.12' + python: '3.13' # Build documentation in the docs/ directory with Sphinx sphinx: @@ -24,10 +24,7 @@ formats: all # Optionally set the version of Python and requirements required to build your docs python: install: - - requirements: requirements-doc.txt - - requirements: requirements-opt.txt - - requirements: requirements.txt - - path: . + - path: .[docs,full] # path to the package relative to the root # Optionally include all submodules submodules: diff --git a/docs/source/install_developers.rst b/docs/source/install_developers.rst index 04e351c41..72da40332 100644 --- a/docs/source/install_developers.rst +++ b/docs/source/install_developers.rst @@ -52,11 +52,11 @@ Option 2: Using conda The `conda package and environment management system`_ is an alternate way of managing virtual environments. First, install Anaconda_ to install the ``conda`` tool. Then create and -activate a new virtual environment called ``"hdmf-env"`` with Python 3.12 installed. +activate a new virtual environment called ``"hdmf-env"`` with Python 3.13 installed. .. code:: bash - conda create --name hdmf-env python=3.12 + conda create --name hdmf-env python=3.13 conda activate hdmf-env Similar to a virtual environment created with ``venv``, a conda environment @@ -88,8 +88,7 @@ package requirements using the pip_ Python package manager, and install HDMF in git clone --recurse-submodules https://github.com/hdmf-dev/hdmf.git cd hdmf - pip install -r requirements.txt -r requirements-dev.txt -r requirements-doc.txt -r requirements-opt.txt - pip install -e . + pip install -e ".[all]" .. note:: diff --git a/docs/source/software_process.rst b/docs/source/software_process.rst index 30501769e..c4971d56e 100644 --- a/docs/source/software_process.rst +++ b/docs/source/software_process.rst @@ -45,48 +45,44 @@ pyproject.toml_ contains a list of package dependencies and their version ranges running HDMF. As a library, upper bound version constraints create more harm than good in the long term (see this `blog post`_) so we avoid setting upper bounds on requirements. -If some of the packages are outdated, see :ref:`update_requirements_files`. +When setting lower bounds, make sure to specify the lower bounds in both pyproject.toml_ and +requirements-min.txt_. The latter is used in automated testing to ensure that the package runs +correctly using the minimum versions of dependencies. + +Minimum requirements should be updated manually if a new feature or bug fix is added in a dependency that is required +for proper running of HDMF. Minimum requirements should also be updated if a user requests that HDMF be installable +with an older version of a dependency, all tests pass using the older version, and there is no valid reason for the +minimum version to be as high as it is. .. _pyproject.toml: https://github.com/hdmf-dev/hdmf/blob/dev/pyproject.toml .. _blog post: https://iscinumpy.dev/post/bound-version-constraints/ +.. _requirements-min.txt: https://github.com/hdmf-dev/hdmf/blob/dev/requirements-min.txt -------------------- Testing Requirements -------------------- -There are several kinds of requirements files used for testing PyNWB. - -The first one is requirements-min.txt_, which lists the package dependencies and their minimum versions for -installing HDMF. +pyproject.toml_ contains the optional dependency group "test" with testing requirements. -The second one is requirements.txt_, which lists the pinned (concrete) dependencies to reproduce -an entire development environment to use HDMF. +See tox.ini_ and the GitHub Actions workflows for how different testing environments are +defined using the optional dependency groups. -The third one is requirements-dev.txt_, which list the pinned (concrete) dependencies to reproduce -an entire development environment to use HDMF, run HDMF tests, check code style, compute coverage, and create test -environments. +environment-ros3.yml_ lists the dependencies used to test ROS3 streaming in HDMF which +can only be done in a Conda environment. -The fourth one is requirements-opt.txt_, which lists the pinned (concrete) optional dependencies to use all -available features in HDMF. - -The final one is environment-ros3.yml_, which lists the dependencies used to -test ROS3 streaming in HDMF. - -.. _requirements-min.txt: https://github.com/hdmf-dev/hdmf/blob/dev/requirements-min.txt -.. _requirements.txt: https://github.com/hdmf-dev/hdmf/blob/dev/requirements.txt -.. _requirements-dev.txt: https://github.com/hdmf-dev/hdmf/blob/dev/requirements-dev.txt -.. _requirements-opt.txt: https://github.com/hdmf-dev/hdmf/blob/dev/requirements-opt.txt +.. _tox.ini: https://github.com/hdmf-dev/hdmf/blob/dev/tox.ini .. _environment-ros3.yml: https://github.com/hdmf-dev/hdmf/blob/dev/environment-ros3.yml -------------------------- Documentation Requirements -------------------------- -requirements-doc.txt_ lists the dependencies to generate the documentation for HDMF. -Both this file and `requirements.txt` are used by ReadTheDocs_ to initialize the local environment for Sphinx to run. +pyproject.toml_ contains the optional dependency group "docs" with documentation requirements. +This dependency group is used by ReadTheDocs_ to initialize the local environment for Sphinx to run +(see .readthedocs.yaml_). -.. _requirements-doc.txt: https://github.com/hdmf-dev/hdmf/blob/dev/requirements-doc.txt .. _ReadTheDocs: https://readthedocs.org/projects/hdmf/ +.. _.readthedocs.yaml: https://github.com/hdmf-dev/hdmf/blob/dev/.readthedocs.yaml ------------------------- Versioning and Releasing diff --git a/docs/source/update_requirements.rst b/docs/source/update_requirements.rst deleted file mode 100644 index 65b4b99d4..000000000 --- a/docs/source/update_requirements.rst +++ /dev/null @@ -1,78 +0,0 @@ - -.. _update_requirements_files: - -================================ -How to Update Requirements Files -================================ - -The different requirements files introduced in :ref:`software_process` section are the following: - -* requirements.txt_ -* requirements-dev.txt_ -* requirements-doc.txt_ -* requirements-min.txt_ -* requirements-opt.txt_ - -.. _requirements.txt: https://github.com/hdmf-dev/hdmf/blob/dev/requirements.txt -.. _requirements-dev.txt: https://github.com/hdmf-dev/hdmf/blob/dev/requirements-dev.txt -.. _requirements-doc.txt: https://github.com/hdmf-dev/hdmf/blob/dev/requirements-doc.txt -.. _requirements-min.txt: https://github.com/hdmf-dev/hdmf/blob/dev/requirements-min.txt -.. _requirements-opt.txt: https://github.com/hdmf-dev/hdmf/blob/dev/requirements-opt.txt - -requirements.txt -================ - -`requirements.txt` of the project can be created or updated and then captured using -the following script: - -.. code:: - - mkvirtualenv hdmf-requirements - - cd hdmf - pip install . - pip check # check for package conflicts - pip freeze > requirements.txt - - deactivate - rmvirtualenv hdmf-requirements - - -requirements-(dev|doc|opt).txt -============================== - -Any of these requirements files can be updated using -the following scripts: - -.. code:: - - cd hdmf - - # Set the requirements file to update - target_requirements=requirements-dev.txt - - mkvirtualenv hdmf-requirements - - # Install updated requirements - pip install -U -r $target_requirements - - # If relevant, you could pip install new requirements now - # pip install -U - - # Check for any conflicts in installed packages - pip check - - # Update list of pinned requirements - pip freeze > $target_requirements - - deactivate - rmvirtualenv hdmf-requirements - - -requirements-min.txt -==================== - -Minimum requirements should be updated manually if a new feature or bug fix is added in a dependency that is required -for proper running of HDMF. Minimum requirements should also be updated if a user requests that HDMF be installable -with an older version of a dependency, all tests pass using the older version, and there is no valid reason for the -minimum version to be as high as it is. diff --git a/environment-ros3.yml b/environment-ros3.yml index be2b9d4f6..3a16cb325 100644 --- a/environment-ros3.yml +++ b/environment-ros3.yml @@ -1,4 +1,4 @@ -# pinned dependencies to reproduce an entire development environment to use PyNWB with ROS3 support +# environment file used to test HDMF with ROS3 support name: ros3 channels: - conda-forge @@ -7,9 +7,8 @@ dependencies: - python==3.13 - h5py==3.12.1 - matplotlib==3.9.2 - - numpy==2.1.3 + - numpy==2.2.1 - pandas==2.2.3 - python-dateutil==2.8.2 - - pytest==8.1.2 # regression introduced in pytest 8.2.*, will be fixed in 8.3.0 - - pytest-cov==5.0.0 - - setuptools + - pytest==8.3.4 + - pytest-cov==6.0.0 diff --git a/pyproject.toml b/pyproject.toml index 86b9e18e1..797bfcecb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,12 +40,38 @@ dependencies = [ dynamic = ["version"] [project.optional-dependencies] -tqdm = ["tqdm>=4.41.0"] -zarr = ["zarr>=2.12.0"] -termset = ["linkml-runtime>=1.5.5", - "schemasheets>=0.1.23", - "oaklib>=0.5.12", - "pyyaml>=6.0.1"] +# all optional dependencies +full = [ + "tqdm>=4.41.0", + "zarr>=2.12.0,<3", + "linkml-runtime>=1.5.5", + "schemasheets>=0.1.23", + "oaklib>=0.5.12", + "pyyaml>=6.0.1", +] + +# development dependencies +test = [ + "codespell", + "pre-commit", + "pytest", + "pytest-cov", + "python-dateutil", + "ruff", + "tox", +] + +# documentation dependencies +docs = [ + "matplotlib", + "sphinx>=4", # improved support for docutils>=0.17 + "sphinx_rtd_theme>=1", # <1 does not work with docutils>=0.17 + "sphinx-gallery", + "sphinx-copybutton", +] + +# all possible dependencies +all = ["hdmf-zarr[full,test,docs]"] [project.urls] "Homepage" = "https://github.com/hdmf-dev/hdmf" diff --git a/requirements-dev.txt b/requirements-dev.txt deleted file mode 100644 index 95cf0797e..000000000 --- a/requirements-dev.txt +++ /dev/null @@ -1,14 +0,0 @@ -# pinned dependencies to reproduce an entire development environment to use HDMF, run HDMF tests, check code style, -# compute coverage, and create test environments. note that depending on the version of python installed, different -# versions of requirements may be installed due to package incompatibilities. -# -black==24.4.2 -codespell==2.3.0 -coverage==7.5.4 -pre-commit==3.7.1; python_version >= "3.9" -pre-commit==3.5.0; python_version < "3.9" -pytest==8.1.2 # regression introduced in pytest 8.2.*, will be fixed in 8.3.0 -pytest-cov==5.0.0 -python-dateutil==2.8.2 -ruff==0.5.0 -tox==4.15.1 diff --git a/requirements-doc.txt b/requirements-doc.txt deleted file mode 100644 index 32a790cf8..000000000 --- a/requirements-doc.txt +++ /dev/null @@ -1,6 +0,0 @@ -# dependencies to generate the documentation for HDMF -matplotlib -sphinx>=4 # improved support for docutils>=0.17 -sphinx_rtd_theme>=1 # <1 does not work with docutils>=0.17 -sphinx-gallery -sphinx-copybutton diff --git a/requirements-opt.txt b/requirements-opt.txt deleted file mode 100644 index 8811ff46e..000000000 --- a/requirements-opt.txt +++ /dev/null @@ -1,6 +0,0 @@ -# pinned dependencies that are optional. used to reproduce an entire development environment to use HDMF -tqdm==4.66.4 -zarr==2.18.2 -linkml-runtime==1.7.7 -schemasheets==0.2.1 -oaklib==0.6.10 diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index b431b4e63..000000000 --- a/requirements.txt +++ /dev/null @@ -1,8 +0,0 @@ -# pinned dependencies to reproduce an entire development environment to use HDMF -h5py==3.12.1 -jsonschema==4.22.0 -numpy==2.1.3 -ruamel.yaml==0.18.2 -pandas==2.2.3 -scipy==1.14.1; python_version >= "3.10" -scipy==1.11.3; python_version < "3.10" diff --git a/tox.ini b/tox.ini index 42305ae25..0a506cab2 100644 --- a/tox.ini +++ b/tox.ini @@ -4,54 +4,54 @@ # and then run "tox -e [envname]" from this directory. [tox] -requires = pip >= 22.0 +requires = pip >= 24.3.1 [testenv] download = True setenv = PYTHONDONTWRITEBYTECODE = 1 - VIRTUALENV_PIP = 23.3.1 recreate = - pinned, minimum, upgraded, prerelease: False + minimum, upgraded, prerelease: False build, wheelinstall: True # good practice to recreate the environment skip_install = - pinned, minimum, upgraded, prerelease, wheelinstall: False + minimum, upgraded, prerelease, wheelinstall: False build: True # no need to install anything when building install_command = # when using [testenv:wheelinstall] and --installpkg, the wheel and its dependencies # are installed, instead of the package in the current directory - pinned, minimum, wheelinstall: python -I -m pip install {opts} {packages} - upgraded: python -I -m pip install -U {opts} {packages} - prerelease: python -I -m pip install -U --pre {opts} {packages} -deps = - # use pinned, minimum, or neither (use dependencies in pyproject.toml) - pytest, gallery: -rrequirements-dev.txt - gallery: -rrequirements-doc.txt - optional: -rrequirements-opt.txt - pinned: -rrequirements.txt - minimum: -rrequirements-min.txt + minimum, wheelinstall: python -I -m pip install {opts} {packages} + upgraded: python -I -m pip install -U {opts} {packages} + prerelease: python -I -m pip install -U --pre {opts} {packages} +deps = + # which requirements files to use (default: none) + minimum: -r requirements-min.txt +extras = + # which optional dependency set(s) to use (default: none) + pytest: test + gallery: doc + optional: full commands = + # commands to run for every environment python --version # print python version for debugging python -m pip check # check for conflicting packages python -m pip list # list installed packages for debugging + + # commands to run for select environments pytest: pytest -v gallery: python test_gallery.py build: python -m pip install -U build build: python -m build wheelinstall: python -c "import hdmf; import hdmf.common" -# list of pre-defined environments. (Technically environments not listed here -# like build-py312 can also be used.) -[testenv:pytest-py313-upgraded] -[testenv:pytest-py313-prerelease] -[testenv:pytest-py311-optional-pinned] # some optional reqs not compatible with py312 yet -[testenv:pytest-py{39,310,311,312,313}-pinned] +# list of pre-defined environments +[testenv:pytest-py{39,310,311,312,313}-upgraded] +[testenv:pytest-py313-upgraded-optional] +[testenv:pytest-py313-prerelease-optional] [testenv:pytest-py39-minimum] -[testenv:gallery-py313-upgraded] -[testenv:gallery-py313-prerelease] -[testenv:gallery-py311-optional-pinned] -[testenv:gallery-py{39,310,311,312,313}-pinned] +[testenv:gallery-py{39,310,311,312,313}-upgraded] +[testenv:gallery-py313-upgraded-optional] +[testenv:gallery-py313-prerelease-optional] [testenv:gallery-py39-minimum] [testenv:build] # using tox for this so that we can have a clean build environment