From 918e6ba29fa7b056d4169f52f8a0f8c7fc9e63f9 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Tue, 15 Aug 2023 12:44:25 -0700 Subject: [PATCH] Fix termset reqs, rename deps tag (#943) --- .github/workflows/check_external_links.yml | 2 +- .github/workflows/run_all_tests.yml | 3 +++ .github/workflows/run_hdmf_zarr_tests.yml | 2 +- .github/workflows/run_pynwb_tests.yml | 2 +- .readthedocs.yaml | 1 + docs/gallery/plot_term_set.py | 5 ++--- pyproject.toml | 8 ++++---- requirements-doc.txt | 4 ---- requirements-min.txt | 9 +++++---- src/hdmf/term_set.py | 19 ++++++++++--------- test_gallery.py | 5 +++++ tests/unit/test_term_set.py | 16 ++++------------ tox.ini | 9 +++++++++ 13 files changed, 46 insertions(+), 39 deletions(-) diff --git a/.github/workflows/check_external_links.yml b/.github/workflows/check_external_links.yml index 1c709ba79..031a26c1c 100644 --- a/.github/workflows/check_external_links.yml +++ b/.github/workflows/check_external_links.yml @@ -28,7 +28,7 @@ jobs: - name: Install Sphinx dependencies and package run: | python -m pip install --upgrade pip - python -m pip install -r requirements-doc.txt + python -m pip install -r requirements-doc.txt -r requirements-opt.txt python -m pip install . - name: Check Sphinx external links diff --git a/.github/workflows/run_all_tests.yml b/.github/workflows/run_all_tests.yml index 59d095c62..3e720f095 100644 --- a/.github/workflows/run_all_tests.yml +++ b/.github/workflows/run_all_tests.yml @@ -94,12 +94,15 @@ jobs: matrix: include: - { name: linux-gallery-python3.8-minimum , test-tox-env: gallery-py38-minimum , python-ver: "3.8" , os: ubuntu-latest } + - { name: linux-gallery-python3.11-optional , test-tox-env: gallery-py311-optional , python-ver: "3.11", os: ubuntu-latest } - { name: linux-gallery-python3.11-upgraded , test-tox-env: gallery-py311-upgraded , python-ver: "3.11", os: ubuntu-latest } - { name: linux-gallery-python3.11-prerelease , test-tox-env: gallery-py311-prerelease, python-ver: "3.11", os: ubuntu-latest } - { name: windows-gallery-python3.8-minimum , test-tox-env: gallery-py38-minimum , python-ver: "3.8" , os: windows-latest } + - { name: windows-gallery-python3.11-optional , test-tox-env: gallery-py311-optional , python-ver: "3.11", os: windows-latest } - { name: windows-gallery-python3.11-upgraded , test-tox-env: gallery-py311-upgraded , python-ver: "3.11", os: windows-latest } - { name: windows-gallery-python3.11-prerelease, test-tox-env: gallery-py311-prerelease, python-ver: "3.11", os: windows-latest } - { name: macos-gallery-python3.8-minimum , test-tox-env: gallery-py38-minimum , python-ver: "3.8" , os: macos-latest } + - { name: macos-gallery-python3.11-optional , test-tox-env: gallery-py311-optional , python-ver: "3.11", os: macos-latest } - { name: macos-gallery-python3.11-upgraded , test-tox-env: gallery-py311-upgraded , python-ver: "3.11", os: macos-latest } - { name: macos-gallery-python3.11-prerelease , test-tox-env: gallery-py311-prerelease, python-ver: "3.11", os: macos-latest } steps: diff --git a/.github/workflows/run_hdmf_zarr_tests.yml b/.github/workflows/run_hdmf_zarr_tests.yml index 63f5bebcb..9221594f4 100644 --- a/.github/workflows/run_hdmf_zarr_tests.yml +++ b/.github/workflows/run_hdmf_zarr_tests.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: jobs: - run-tests: + run-hdmf-zarr-tests: runs-on: ubuntu-latest steps: - name: Cancel non-latest runs diff --git a/.github/workflows/run_pynwb_tests.yml b/.github/workflows/run_pynwb_tests.yml index 5e250cbf7..2578e5383 100644 --- a/.github/workflows/run_pynwb_tests.yml +++ b/.github/workflows/run_pynwb_tests.yml @@ -6,7 +6,7 @@ on: workflow_dispatch: jobs: - run-tests: + run-pynwb-tests: runs-on: ubuntu-latest steps: - name: Cancel non-latest runs diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 99338f5f5..a4f1ea037 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -25,6 +25,7 @@ formats: all python: install: - requirements: requirements-doc.txt + - requirements: requirements-opt.txt - requirements: requirements.txt - path: . diff --git a/docs/gallery/plot_term_set.py b/docs/gallery/plot_term_set.py index fc065264b..889fb86ea 100644 --- a/docs/gallery/plot_term_set.py +++ b/docs/gallery/plot_term_set.py @@ -63,12 +63,11 @@ """ from hdmf.common import DynamicTable, VectorData import os -import sys try: import linkml_runtime # noqa: F401 -except ImportError: - sys.exit(0) +except ImportError as e: + raise ImportError("Please install linkml-runtime to run this example: pip install linkml-runtime") from e from hdmf.term_set import TermSet try: diff --git a/pyproject.toml b/pyproject.toml index d834ea8f5..ee8037be5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,10 +42,10 @@ dynamic = ["version"] [project.optional-dependencies] zarr = ["zarr>=2.12.0"] tqdm = ["tqdm>=4.41.0"] -termset_reqs = ["linkml-runtime<=1.5.5; python_version >= '3.9'", - "schemasheets>=0.1.23; python_version >= '3.9'", - "oaklib>=0.5.12; python_version >= '3.9'", - "pyyaml>=6.0.1; python_version >= '3.9'"] +termset = ["linkml-runtime>=1.5.5; python_version >= '3.9'", + "schemasheets>=0.1.23; python_version >= '3.9'", + "oaklib>=0.5.12; python_version >= '3.9'", + "pyyaml>=6.0.1; python_version >= '3.9'"] [project.urls] "Homepage" = "https://github.com/hdmf-dev/hdmf" diff --git a/requirements-doc.txt b/requirements-doc.txt index 11ca9fb97..32a790cf8 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -4,7 +4,3 @@ sphinx>=4 # improved support for docutils>=0.17 sphinx_rtd_theme>=1 # <1 does not work with docutils>=0.17 sphinx-gallery sphinx-copybutton -linkml-runtime==1.5.5; python_version >= "3.9" -schemasheets==0.1.23; python_version >= "3.9" -oaklib==0.5.12; python_version >= "3.9" -pyyaml==6.0.1; python_version >= "3.9" diff --git a/requirements-min.txt b/requirements-min.txt index e27b12c14..a437fc588 100644 --- a/requirements-min.txt +++ b/requirements-min.txt @@ -6,9 +6,10 @@ numpy==1.18 pandas==1.0.5 # when this is changed to >=1.5.0, see TODO items referenced in #762 ruamel.yaml==0.16 scipy==1.4 -linkml-runtime==1.5.5; python_version >= "3.9" -schemasheets==0.1.23; python_version >= "3.9" -oaklib==0.5.12; python_version >= "3.9" -pyyaml==6.0.1; python_version >= "3.9" +# this file is currently used to test only python~=3.8 so these dependencies are not needed +# linkml-runtime==1.5.5; python_version >= "3.9" +# schemasheets==0.1.23; python_version >= "3.9" +# oaklib==0.5.12; python_version >= "3.9" +# pyyaml==6.0.1; python_version >= "3.9" tqdm==4.41.0 zarr==2.12.0 diff --git a/src/hdmf/term_set.py b/src/hdmf/term_set.py index 9b5983b56..b2b59dfd0 100644 --- a/src/hdmf/term_set.py +++ b/src/hdmf/term_set.py @@ -5,12 +5,12 @@ import warnings -class TermSet(): +class TermSet: """ Class for implementing term sets from ontologies and other resources used to define the meaning and/or identify of terms. - :ivar term_schema_path: The path to LinkML YAML enumeration schema + :ivar term_schema_path: The path to the LinkML YAML enumeration schema :ivar sources: The prefixes for the ontologies used in the TermSet :ivar view: SchemaView of the term set schema :ivar schemasheets_folder: The path to the folder containing the LinkML TSV files @@ -22,7 +22,7 @@ def __init__(self, dynamic: bool=False ): """ - :param term_schema_path: The path to LinkML YAML enumeration schema + :param term_schema_path: The path to the LinkML YAML enumeration schema :param schemasheets_folder: The path to the folder containing the LinkML TSV files :param dynamic: Boolean parameter denoting whether the schema uses Dynamic Enumerations @@ -132,8 +132,8 @@ def __schemasheets_convert(self): from linkml_runtime.utils.schema_as_dict import schema_as_dict from schemasheets.schemamaker import SchemaMaker except ImportError: # pragma: no cover - msg="Install schemasheets." # pragma: no cover - raise ValueError(msg) # pragma: no cover + msg = "Install schemasheets." + raise ValueError(msg) schema_maker = SchemaMaker() tsv_file_paths = glob.glob(self.schemasheets_folder + "/*.tsv") schema = schema_maker.create_schema(tsv_file_paths) @@ -154,11 +154,12 @@ def __enum_expander(self): This method returns a path to the new schema to be viewed via SchemaView. """ try: - warnings.filterwarnings("ignore", category=DeprecationWarning) - from oaklib.utilities.subsets.value_set_expander import ValueSetExpander + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", category=DeprecationWarning) + from oaklib.utilities.subsets.value_set_expander import ValueSetExpander except ImportError: # pragma: no cover - msg = 'Install oaklib.' # pragma: no cover - raise ValueError(msg) # pragma: no cover + msg = 'Install oaklib.' + raise ValueError(msg) expander = ValueSetExpander() # TODO: linkml should raise a warning if the schema does not have dynamic enums enum = list(self.view.all_enums()) diff --git a/test_gallery.py b/test_gallery.py index dc03acdb2..cb77ac430 100644 --- a/test_gallery.py +++ b/test_gallery.py @@ -71,6 +71,11 @@ def run_gallery_tests(): category=RuntimeWarning, ) _import_from_file(script) + except (ImportError, ValueError) as e: + if "linkml" in str(e) and sys.version_info < (3, 9): + pass # this is OK because plot_term_set.py and plot_external_resources.py cannot be run on Python 3.8 + else: + raise e except Exception: print(traceback.format_exc()) FAILURES += 1 diff --git a/tests/unit/test_term_set.py b/tests/unit/test_term_set.py index 8130c7e4b..2acaa7954 100644 --- a/tests/unit/test_term_set.py +++ b/tests/unit/test_term_set.py @@ -1,5 +1,4 @@ import os -import unittest from hdmf.term_set import TermSet from hdmf.testing import TestCase, remove_test_file @@ -19,48 +18,44 @@ class TestTermSet(TestCase): - @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed") + def setUp(self): + if not REQUIREMENTS_INSTALLED: + self.skipTest("optional LinkML module is not installed") + def test_termset_setup(self): termset = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') self.assertEqual(list(termset.sources), ['NCBI_TAXON']) - @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed") def test_view_set(self): termset = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') expected = ['Homo sapiens', 'Mus musculus', 'Ursus arctos horribilis', 'Myrmecophaga tridactyla'] self.assertEqual(list(termset.view_set), expected) self.assertIsInstance(termset.view, SchemaView) - @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed") def test_termset_validate(self): termset = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') self.assertEqual(termset.validate('Homo sapiens'), True) - @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed") def test_termset_validate_false(self): termset = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') self.assertEqual(termset.validate('missing_term'), False) - @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed") def test_get_item(self): termset = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') self.assertEqual(termset['Homo sapiens'].id, 'NCBI_TAXON:9606') self.assertEqual(termset['Homo sapiens'].description, 'the species is human') self.assertEqual(termset['Homo sapiens'].meaning, 'https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=9606') - @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed") def test_get_item_key_error(self): termset = TermSet(term_schema_path='tests/unit/example_test_term_set.yaml') with self.assertRaises(ValueError): termset['Homo Ssapiens'] - @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed") def test_schema_sheets_and_path_provided_error(self): folder = os.path.join(CUR_DIR, "test_term_set_input", "schemasheets") with self.assertRaises(ValueError): TermSet(term_schema_path='tests/unit/example_test_term_set.yaml', schemasheets_folder=folder) - @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed") def test_view_set_sheets(self): folder = os.path.join(CUR_DIR, "test_term_set_input", "schemasheets") termset = TermSet(schemasheets_folder=folder) @@ -69,7 +64,6 @@ def test_view_set_sheets(self): self.assertEqual(list(termset.view_set), expected) self.assertIsInstance(termset.view, SchemaView) - @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed") def test_enum_expander(self): schema_path = 'tests/unit/example_dynamic_term_set.yaml' termset = TermSet(term_schema_path=schema_path, dynamic=True) @@ -90,7 +84,6 @@ def test_enum_expander(self): filename = os.path.splitext(os.path.basename(schema_path))[0] remove_test_file(f"tests/unit/expanded_{filename}.yaml") - @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed") def test_enum_expander_output(self): schema_path = 'tests/unit/example_dynamic_term_set.yaml' termset = TermSet(term_schema_path=schema_path, dynamic=True) @@ -102,7 +95,6 @@ def test_enum_expander_output(self): self.assertEqual(convert_path, expected_path) - @unittest.skipIf(not REQUIREMENTS_INSTALLED, "optional LinkML module is not installed") def test_folder_output(self): folder = os.path.join(CUR_DIR, "test_term_set_input", "schemasheets") termset = TermSet(schemasheets_folder=folder) diff --git a/tox.ini b/tox.ini index 9f6114972..596262002 100644 --- a/tox.ini +++ b/tox.ini @@ -152,6 +152,15 @@ basepython = python3.11 deps = {[testenv:gallery]deps} commands = {[testenv:gallery]commands} +[testenv:gallery-py311-optional] +basepython = python3.11 +deps = + -rrequirements-dev.txt + -rrequirements.txt + -rrequirements-doc.txt + -rrequirements-opt.txt +commands = {[testenv:gallery]commands} + # Test with python 3.11; pinned dev, doc, and optional reqs; upgraded run reqs [testenv:gallery-py311-upgraded] basepython = python3.11