From 398927ada98775b76d1282b233ddb932718a2b2a Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Thu, 10 Oct 2024 16:01:07 -0500 Subject: [PATCH 1/3] Fix topological sort logic (#1162) * Fix topo sort * Update changelog --- CHANGELOG.md | 3 ++- src/spyglass/utils/dj_graph.py | 35 ++++++++++++++++++++++++++++++---- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 63aef8498..0140d231c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,8 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Add docstrings to all public methods #1076 - Update DataJoint to 0.14.2 #1081 - Allow restriction based on parent keys in `Merge.fetch_nwb()` #1086, #1126 -- Import `datajoint.dependencies.unite_master_parts` -> `topo_sort` #1116, #1137 +- Import `datajoint.dependencies.unite_master_parts` -> `topo_sort` #1116, + #1137, #1162 - Fix bool settings imported from dj config file #1117 - Allow definition of tasks and new probe entries from config #1074, #1120 - Enforce match between ingested nwb probe geometry and existing table entry diff --git a/src/spyglass/utils/dj_graph.py b/src/spyglass/utils/dj_graph.py index a66a6ac5d..625202569 100644 --- a/src/spyglass/utils/dj_graph.py +++ b/src/spyglass/utils/dj_graph.py @@ -17,6 +17,7 @@ from datajoint.user_tables import TableMeta from datajoint.utils import get_master, to_camel_case from networkx import ( + DiGraph, NetworkXNoPath, NodeNotFound, all_simple_paths, @@ -33,10 +34,36 @@ unique_dicts, ) -try: # Datajoint 0.14.2+ uses topo_sort instead of unite_master_parts - from datajoint.dependencies import topo_sort as dj_topo_sort -except ImportError: - from datajoint.dependencies import unite_master_parts as dj_topo_sort + +def dj_topo_sort(graph: DiGraph) -> List[str]: + """Topologically sort graph. + + Uses datajoint's topo_sort if available, otherwise uses networkx's + topological_sort, combined with datajoint's unite_master_parts. + + NOTE: This ordering will impact _hash_upstream, but usage should be + consistent before/after a no-transaction populate. + + Parameters + ---------- + graph : nx.DiGraph + Directed graph to sort + + Returns + ------- + List[str] + List of table names in topological order + """ + __import__("pdb").set_trace() + try: # Datajoint 0.14.2+ uses topo_sort instead of unite_master_parts + from datajoint.dependencies import topo_sort + + return topo_sort(graph) + except ImportError: + from datajoint.dependencies import unite_master_parts + from networkx.algorithms.dag import topological_sort + + return unite_master_parts(list(topological_sort(graph))) class Direction(Enum): From 6be6b71ce9068c9853844a1a369893b2956905bb Mon Sep 17 00:00:00 2001 From: Chris Broz Date: Tue, 15 Oct 2024 14:09:15 -0500 Subject: [PATCH 2/3] Fix tests (#1165) * Fix tests, run test on PR * Update changelog * Inherit secrets in PR tests run * Revert run on PR --- .github/workflows/test-conda.yml | 10 +++++----- CHANGELOG.md | 6 ++++-- src/spyglass/utils/dj_graph.py | 1 - 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test-conda.yml b/.github/workflows/test-conda.yml index 808417967..d34e83c01 100644 --- a/.github/workflows/test-conda.yml +++ b/.github/workflows/test-conda.yml @@ -2,7 +2,7 @@ name: Tests on: push: - branches: + branches: - '!test_branch' - '!documentation' schedule: # once a day at midnight UTC @@ -53,7 +53,7 @@ jobs: sudo apt-get install mysql-client libmysqlclient-dev libgirepository1.0-dev -y sudo apt-get install ffmpeg libsm6 libxext6 -y # non-dlc position deps - name: Run pip install for test deps - run: | + run: | pip install --quiet .[test] - name: Download data env: @@ -61,8 +61,8 @@ jobs: NWBFILE: minirec20230622.nwb # Relative to Base URL VID_ONE: 20230622_sample_01_a1/20230622_sample_01_a1.1.h264 VID_TWO: 20230622_sample_02_a1/20230622_sample_02_a1.1.h264 - RAW_DIR: /home/runner/work/spyglass/spyglass/tests/_data/raw/ - VID_DIR: /home/runner/work/spyglass/spyglass/tests/_data/video/ + RAW_DIR: /home/runner/work/spyglass/spyglass/tests/_data/raw/ + VID_DIR: /home/runner/work/spyglass/spyglass/tests/_data/video/ run: | mkdir -p $RAW_DIR $VID_DIR wget_opts() { # Declare func with download options @@ -76,4 +76,4 @@ jobs: wget_opts $VID_DIR $VID_TWO - name: Run tests run: | - pytest --no-docker --no-dlc + pytest --no-docker --no-dlc diff --git a/CHANGELOG.md b/CHANGELOG.md index 0140d231c..2c3fbe4de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,8 +30,10 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Update DataJoint install and password instructions #1131 - Fix dandi upload process for nwb's with video or linked objects #1095, #1151 - Minor docs fixes #1145 -- Remove stored hashes from pytests #1152 -- Remove mambaforge from tests #1153 +- Test fixes + - Remove stored hashes from pytests #1152 + - Remove mambaforge from tests #1153 + - Remove debug statement #1164 ### Pipelines diff --git a/src/spyglass/utils/dj_graph.py b/src/spyglass/utils/dj_graph.py index 625202569..435d37d38 100644 --- a/src/spyglass/utils/dj_graph.py +++ b/src/spyglass/utils/dj_graph.py @@ -54,7 +54,6 @@ def dj_topo_sort(graph: DiGraph) -> List[str]: List[str] List of table names in topological order """ - __import__("pdb").set_trace() try: # Datajoint 0.14.2+ uses topo_sort instead of unite_master_parts from datajoint.dependencies import topo_sort From e57638e3c928c816a8f9361100b49021f77cb832 Mon Sep 17 00:00:00 2001 From: Eric Denovellis Date: Wed, 23 Oct 2024 10:10:52 -0700 Subject: [PATCH 3/3] Allow python <3.13 and remove numpy pin (#1169) * Increase upper bound of python to include 3.12 * Add tests for different python versions * Minor formatting * Use quotes for version to avoid parsing errors * Ensure build dependencies are installed distutils was removed for python 3.12 * Add more build dependencies * Try different resources * Temp test on <3.12 * Remove importlib_metadata pin Fixed in twine https://github.com/pypa/twine/issues/1125 * Revert "Temp test on <3.12" This reverts commit 5b2736651928e01e38816a4a7f479415ea41fc74. * Change install order of build dependencies * Remove numpy pin * Update CHANGELOG.md --- .github/workflows/test-package-build.yml | 10 ++++--- CHANGELOG.md | 3 ++ environment.yml | 4 +-- environment_dlc.yml | 4 +-- pyproject.toml | 36 ++++++++++++------------ 5 files changed, 31 insertions(+), 26 deletions(-) diff --git a/.github/workflows/test-package-build.yml b/.github/workflows/test-package-build.yml index 3513bb664..e07017a31 100644 --- a/.github/workflows/test-package-build.yml +++ b/.github/workflows/test-package-build.yml @@ -29,7 +29,6 @@ jobs: python-version: 3.9 - run: | pip install --upgrade build twine - pip install importlib_metadata==7.2.1 # twine #977 - name: Build sdist and wheel run: python -m build - run: twine check dist/* @@ -50,6 +49,7 @@ jobs: needs: [build] strategy: matrix: + python-version: ['3.9', '3.10', '3.11', '3.12'] package: ['wheel', 'sdist', 'archive'] steps: - name: Download sdist and wheel artifacts @@ -66,11 +66,13 @@ jobs: path: archive/ - uses: actions/setup-python@v5 with: - python-version: 3.9 + python-version: ${{ matrix.python-version }} - name: Display Python version run: python -c "import sys; print(sys.version)" - - name: Update pip - run: pip install --upgrade pip + - name: Install build dependencies + run: | + pip install --upgrade setuptools wheel + pip install --upgrade pip - name: Install wheel if: matrix.package == 'wheel' run: pip install dist/*.whl diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c3fbe4de..84d6d2c11 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,9 @@ dj.FreeTable(dj.conn(), "common_session.session_group").drop() - Remove stored hashes from pytests #1152 - Remove mambaforge from tests #1153 - Remove debug statement #1164 +- Allow python < 3.13 #1169 +- Remove numpy version restriction #1169 +- Add testing for python versions 3.9, 3.10, 3.11, 3.12 #1169 ### Pipelines diff --git a/environment.yml b/environment.yml index 7fa1b51ea..a5229b8a8 100644 --- a/environment.yml +++ b/environment.yml @@ -23,13 +23,13 @@ dependencies: # - libgcc # dlc-only - matplotlib - non_local_detector - - numpy<1.24 + - numpy - pip - position_tools - pybind11 # req by mountainsort4 -> isosplit5 - pydotplus - pyfftw<=0.12.0 # ghostipy req. install from conda-forge for Mac ARM - - python>=3.9,<3.10 + - python>=3.9,<3.13 - pytorch<1.12.0 - ripple_detection - seaborn diff --git a/environment_dlc.yml b/environment_dlc.yml index 9870a0424..156bed793 100644 --- a/environment_dlc.yml +++ b/environment_dlc.yml @@ -23,13 +23,13 @@ dependencies: - libgcc # dlc-only - matplotlib - non_local_detector - - numpy<1.24 + - numpy - pip>=20.2.* - position_tools - pybind11 # req by mountainsort4 -> isosplit5 - pydotplus>=2.0.* - pyfftw<=0.12.0 # ghostipy req. install from conda-forge for Mac ARM - - python>=3.9,<3.10 + - python>=3.9,<3.13 - pytorch<1.12.0 - ripple_detection - seaborn diff --git a/pyproject.toml b/pyproject.toml index ed3a570c8..0bd9164cb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,7 @@ build-backend = "hatchling.build" name = "spyglass-neuro" description = "Neuroscience data analysis framework for reproducible research" readme = "README.md" -requires-python = ">=3.9,<3.10" +requires-python = ">=3.9,<3.13" license = { file = "LICENSE" } authors = [ { name = "Loren Frank", email = "loren.frank@ucsf.edu" }, @@ -46,11 +46,11 @@ dependencies = [ "matplotlib", "ndx_franklab_novela>=0.1.0", "non_local_detector", - "numpy<1.24", + "numpy", "opencv-python", - "panel>=1.4.0", # panel #6325 resolved + "panel>=1.4.0", # panel #6325 resolved "position_tools>=0.1.0", - "pubnub<6.4.0", # TODO: remove this when sortingview is updated + "pubnub<6.4.0", # TODO: remove this when sortingview is updated "pydotplus", "pynwb>=2.2.0,<3", "ripple_detection", @@ -62,21 +62,21 @@ dependencies = [ [project.optional-dependencies] dlc = [ - "ffmpeg", - "deeplabcut[tf]", # removing dlc pin removes need to pin tf/numba + "ffmpeg", + "deeplabcut[tf]", # removing dlc pin removes need to pin tf/numba ] test = [ - "click", # for CLI subpackage only - "docker", # for tests in a container + "click", # for CLI subpackage only + "docker", # for tests in a container "ghostipy", - "kachery", # database access + "kachery", # database access "kachery-client", "kachery-cloud>=0.4.0", "opencv-python-headless", # for headless testing of Qt - "pre-commit", # linting - "pytest", # unit testing - "pytest-cov", # code coverage - "pytest-xvfb", # for headless testing of Qt + "pre-commit", # linting + "pytest", # unit testing + "pytest-cov", # code coverage + "pytest-xvfb", # for headless testing of Qt ] docs = [ "hatch", # Get version from env @@ -134,7 +134,7 @@ addopts = [ # "--no-dlc", # don't run DLC tests "--show-capture=no", "--pdbcls=IPython.terminal.debugger:TerminalPdb", # use ipython debugger - "--doctest-modules", # run doctests in all modules + "--doctest-modules", # run doctests in all modules "--cov=spyglass", "--cov-report=term-missing", "--no-cov-on-fail", @@ -143,9 +143,9 @@ testpaths = ["tests"] log_level = "INFO" env = [ "QT_QPA_PLATFORM = offscreen", # QT fails headless without this - "DISPLAY = :0", # QT fails headless without this - "TF_ENABLE_ONEDNN_OPTS = 0", # TF disable approx calcs - "TF_CPP_MIN_LOG_LEVEL = 2", # Disable TF warnings + "DISPLAY = :0", # QT fails headless without this + "TF_ENABLE_ONEDNN_OPTS = 0", # TF disable approx calcs + "TF_CPP_MIN_LOG_LEVEL = 2", # Disable TF warnings ] [tool.coverage.run] @@ -175,4 +175,4 @@ omit = [ # which submodules have no tests line-length = 80 [tool.ruff.lint] -ignore = ["F401" , "E402", "E501"] +ignore = ["F401", "E402", "E501"]