From 4d131a0d93f26f483484ae776a33a6d018e0d391 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 16:11:31 -0700 Subject: [PATCH 1/7] [pre-commit.ci] pre-commit autoupdate (#1120) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ryan Ly --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f80639c7..3f4d13e28 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: # hooks: # - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.4 + rev: v0.4.7 hooks: - id: ruff # - repo: https://github.com/econchick/interrogate @@ -26,7 +26,7 @@ repos: # hooks: # - id: interrogate - repo: https://github.com/codespell-project/codespell - rev: v2.2.6 + rev: v2.3.0 hooks: - id: codespell additional_dependencies: From 543935f164d5e8beb77bb4d854b1f1a8b1b4b18b Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Thu, 6 Jun 2024 08:13:47 -0700 Subject: [PATCH 2/7] Update CHANGELOG.md (#1124) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f11fa143..f022ef0e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # HDMF Changelog -## HDMF 3.14.1 (Upcoming) +## HDMF 3.14.1 (June 6, 2024) ### Bug fixes - Excluded unnecessary artifacts from sdist and wheel. @rly [#1119](https://github.com/hdmf-dev/hdmf/pull/1119) From ece2c27d9c828f4ecb5e2ee3b66dd49e547f7b4c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 22:26:04 -0700 Subject: [PATCH 3/7] [pre-commit.ci] pre-commit autoupdate (#1127) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f4d13e28..7684975ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: # hooks: # - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.7 + rev: v0.4.8 hooks: - id: ruff # - repo: https://github.com/econchick/interrogate From d756abeb98ef89fde37a71ed64ec84de627f8bfb Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 12 Jun 2024 12:41:51 -0400 Subject: [PATCH 4/7] Fix iterator increment (#1128) --- CHANGELOG.md | 7 +++++++ src/hdmf/data_utils.py | 10 +++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f022ef0e7..ae753b98b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # HDMF Changelog +## HDMF 3.14.2 (???) + +### Bug fixes +- Fix iterator increment causing an extra +1 added after the end of completion. @CodyCBakerPhD [#1128](https://github.com/hdmf-dev/hdmf/pull/1128) + + + ## HDMF 3.14.1 (June 6, 2024) ### Bug fixes diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index c03665caa..f4ac6541f 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -386,14 +386,18 @@ def __next__(self): :returns: DataChunk object with the data and selection of the current buffer. :rtype: DataChunk """ - if self.display_progress: - self.progress_bar.update(n=1) try: buffer_selection = next(self.buffer_selection_generator) + + # Only update after successful iteration + if self.display_progress: + self.progress_bar.update(n=1) + return DataChunk(data=self._get_data(selection=buffer_selection), selection=buffer_selection) except StopIteration: + # Allow text to be written to new lines after completion if self.display_progress: - self.progress_bar.write("\n") # Allows text to be written to new lines after completion + self.progress_bar.write("\n") raise StopIteration def __reduce__(self) -> Tuple[Callable, Iterable]: From 7426275cacc769a10ffca89836765df1355ba9db Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 23:40:49 -0400 Subject: [PATCH 5/7] [pre-commit.ci] pre-commit autoupdate (#1131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.8 → v0.4.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.8...v0.4.9) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7684975ab..0f486273b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: # hooks: # - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.8 + rev: v0.4.9 hooks: - id: ruff # - repo: https://github.com/econchick/interrogate From 539ecf47ad1ad70e23666f7a7d750d2d84535632 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Fri, 28 Jun 2024 16:23:06 -0700 Subject: [PATCH 6/7] Zarr append for datasets (non-reference) (#1136) * Zarr append * Update CHANGELOG.md * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update validator.py * Update testcase.py * Update objectmapper.py * Update h5tools.py * Update h5tools.py * Update h5tools.py * Update src/hdmf/validate/validator.py Co-authored-by: Steph Prince <40640337+stephprince@users.noreply.github.com> --------- Co-authored-by: Steph Prince <40640337+stephprince@users.noreply.github.com> --- CHANGELOG.md | 5 +++-- pyproject.toml | 8 ++++---- src/hdmf/backends/hdf5/h5tools.py | 2 +- src/hdmf/build/objectmapper.py | 2 +- src/hdmf/data_utils.py | 5 ++++- src/hdmf/testing/testcase.py | 2 +- src/hdmf/validate/validator.py | 2 +- tests/unit/utils_test/test_data_utils.py | 14 ++++++++++++++ 8 files changed, 29 insertions(+), 11 deletions(-) create mode 100644 tests/unit/utils_test/test_data_utils.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ae753b98b..5f5db4918 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,12 @@ # HDMF Changelog -## HDMF 3.14.2 (???) +## HDMF 3.14.2 (Upcoming) ### Bug fixes - Fix iterator increment causing an extra +1 added after the end of completion. @CodyCBakerPhD [#1128](https://github.com/hdmf-dev/hdmf/pull/1128) - +### Enhancements +- Support appending to zarr arrays. @mavaylon1 [#1136](https://github.com/hdmf-dev/hdmf/pull/1136) ## HDMF 3.14.1 (June 6, 2024) diff --git a/pyproject.toml b/pyproject.toml index 3a0034087..a089113c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,12 +36,12 @@ dependencies = [ "pandas>=1.0.5", "ruamel.yaml>=0.16", "scipy>=1.4", + "zarr >= 2.12.0", "importlib-resources; python_version < '3.9'", # TODO: remove when minimum python version is 3.9 ] dynamic = ["version"] [project.optional-dependencies] -zarr = ["zarr>=2.12.0"] tqdm = ["tqdm>=4.41.0"] termset = ["linkml-runtime>=1.5.5; python_version >= '3.9'", "schemasheets>=0.1.23; python_version >= '3.9'", @@ -117,7 +117,7 @@ omit = [ # force-exclude = "src/hdmf/common/hdmf-common-schema|docs/gallery" [tool.ruff] -select = ["E", "F", "T100", "T201", "T203"] +lint.select = ["E", "F", "T100", "T201", "T203"] exclude = [ ".git", ".tox", @@ -132,11 +132,11 @@ exclude = [ ] line-length = 120 -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "docs/gallery/*" = ["E402", "T201"] "src/*/__init__.py" = ["F401"] "setup.py" = ["T201"] "test_gallery.py" = ["T201"] -[tool.ruff.mccabe] +[tool.ruff.lint.mccabe] max-complexity = 17 diff --git a/src/hdmf/backends/hdf5/h5tools.py b/src/hdmf/backends/hdf5/h5tools.py index 0604881bb..8135d75e7 100644 --- a/src/hdmf/backends/hdf5/h5tools.py +++ b/src/hdmf/backends/hdf5/h5tools.py @@ -728,7 +728,7 @@ def __read_dataset(self, h5obj, name=None): def _check_str_dtype(self, h5obj): dtype = h5obj.dtype if dtype.kind == 'O': - if dtype.metadata.get('vlen') == str and H5PY_3: + if dtype.metadata.get('vlen') is str and H5PY_3: return StrDataset(h5obj, None) return h5obj diff --git a/src/hdmf/build/objectmapper.py b/src/hdmf/build/objectmapper.py index fed678d41..52fbfec49 100644 --- a/src/hdmf/build/objectmapper.py +++ b/src/hdmf/build/objectmapper.py @@ -1164,7 +1164,7 @@ def __get_subspec_values(self, builder, spec, manager): if not isinstance(builder, DatasetBuilder): # pragma: no cover raise ValueError("__get_subspec_values - must pass DatasetBuilder with DatasetSpec") if (spec.shape is None and getattr(builder.data, 'shape', None) == (1,) and - type(builder.data[0]) != np.void): + type(builder.data[0]) is not np.void): # if a scalar dataset is expected and a 1-element non-compound dataset is given, then read the dataset builder['data'] = builder.data[0] # use dictionary reference instead of .data to bypass error ret[spec] = self.__check_ref_resolver(builder.data) diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index f4ac6541f..71f5bdf6d 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -5,17 +5,20 @@ from warnings import warn from typing import Tuple from itertools import product, chain +from zarr import Array as ZarrArray import h5py import numpy as np from .utils import docval, getargs, popargs, docval_macro, get_data_shape - def append_data(data, arg): if isinstance(data, (list, DataIO)): data.append(arg) return data + elif isinstance(data, ZarrArray): + data.append([arg], axis=0) + return data elif type(data).__name__ == 'TermSetWrapper': # circular import data.append(arg) return data diff --git a/src/hdmf/testing/testcase.py b/src/hdmf/testing/testcase.py index 798df6fe4..1be4bcecd 100644 --- a/src/hdmf/testing/testcase.py +++ b/src/hdmf/testing/testcase.py @@ -174,7 +174,7 @@ def _assert_array_equal(self, :param message: custom additional message to show when assertions as part of this assert are failing """ array_data_types = tuple([i for i in get_docval_macro('array_data') - if (i != list and i != tuple and i != AbstractDataChunkIterator)]) + if (i is not list and i is not tuple and i is not AbstractDataChunkIterator)]) # We construct array_data_types this way to avoid explicit dependency on h5py, Zarr and other # I/O backends. Only list and tuple do not support [()] slicing, and AbstractDataChunkIterator # should never occur here. The effective value of array_data_types is then: diff --git a/src/hdmf/validate/validator.py b/src/hdmf/validate/validator.py index bdfc15f8f..e39011d9f 100644 --- a/src/hdmf/validate/validator.py +++ b/src/hdmf/validate/validator.py @@ -164,7 +164,7 @@ def get_type(data, builder_dtype=None): # Empty array else: # Empty string array - if data.dtype.metadata["vlen"] == str: + if data.dtype.metadata["vlen"] is str: return "utf", None # Undetermined variable length data type. else: # pragma: no cover diff --git a/tests/unit/utils_test/test_data_utils.py b/tests/unit/utils_test/test_data_utils.py new file mode 100644 index 000000000..2e0df7ba8 --- /dev/null +++ b/tests/unit/utils_test/test_data_utils.py @@ -0,0 +1,14 @@ +from hdmf.data_utils import append_data +from hdmf.testing import TestCase + +import numpy as np +from numpy.testing import assert_array_equal +import zarr + +class TestAppendData(TestCase): + + def test_append_data_zarr(self): + zarr_array = zarr.array([1,2,3]) + new = append_data(zarr_array, 4) + + assert_array_equal(new[:], np.array([1,2,3,4])) From 5e13d64efabedb039ed4eb49abd4a9c9e848f72a Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Mon, 1 Jul 2024 12:33:29 -0400 Subject: [PATCH 7/7] Warn when unexpected keys in specs (#1134) * Warn when unexpected keys in specs * Update changelog --- CHANGELOG.md | 7 ++++--- src/hdmf/spec/spec.py | 4 ++++ tests/unit/spec_tests/test_attribute_spec.py | 12 ++++++++++++ tests/unit/spec_tests/test_dataset_spec.py | 12 ++++++++++++ tests/unit/spec_tests/test_group_spec.py | 10 ++++++++++ tests/unit/spec_tests/test_link_spec.py | 12 ++++++++++++ 6 files changed, 54 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f5db4918..74e048a19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,13 @@ ## HDMF 3.14.2 (Upcoming) -### Bug fixes -- Fix iterator increment causing an extra +1 added after the end of completion. @CodyCBakerPhD [#1128](https://github.com/hdmf-dev/hdmf/pull/1128) - ### Enhancements +- Warn when unexpected keys are present in specs. @rly [#1134](https://github.com/hdmf-dev/hdmf/pull/1134) - Support appending to zarr arrays. @mavaylon1 [#1136](https://github.com/hdmf-dev/hdmf/pull/1136) +### Bug fixes +- Fix iterator increment causing an extra +1 added after the end of completion. @CodyCBakerPhD [#1128](https://github.com/hdmf-dev/hdmf/pull/1128) + ## HDMF 3.14.1 (June 6, 2024) ### Bug fixes diff --git a/src/hdmf/spec/spec.py b/src/hdmf/spec/spec.py index 1a6e8d987..6d7d29e49 100644 --- a/src/hdmf/spec/spec.py +++ b/src/hdmf/spec/spec.py @@ -93,9 +93,13 @@ def build_spec(cls, spec_dict): vargs = cls.build_const_args(spec_dict) kwargs = dict() # iterate through the Spec docval and construct kwargs based on matching values in spec_dict + unused_vargs = list(vargs) for x in get_docval(cls.__init__): if x['name'] in vargs: kwargs[x['name']] = vargs.get(x['name']) + unused_vargs.remove(x['name']) + if unused_vargs: + warn(f'Unexpected keys {unused_vargs} in spec {spec_dict}') return cls(**kwargs) diff --git a/tests/unit/spec_tests/test_attribute_spec.py b/tests/unit/spec_tests/test_attribute_spec.py index 15102e728..bac8e12a3 100644 --- a/tests/unit/spec_tests/test_attribute_spec.py +++ b/tests/unit/spec_tests/test_attribute_spec.py @@ -91,3 +91,15 @@ def test_build_spec_no_doc(self): msg = "AttributeSpec.__init__: missing argument 'doc'" with self.assertRaisesWith(TypeError, msg): AttributeSpec.build_spec(spec_dict) + + def test_build_warn_extra_args(self): + spec_dict = { + 'name': 'attribute1', + 'doc': 'test attribute', + 'dtype': 'int', + 'quantity': '?', + } + msg = ("Unexpected keys ['quantity'] in spec {'name': 'attribute1', 'doc': 'test attribute', " + "'dtype': 'int', 'quantity': '?'}") + with self.assertWarnsWith(UserWarning, msg): + AttributeSpec.build_spec(spec_dict) diff --git a/tests/unit/spec_tests/test_dataset_spec.py b/tests/unit/spec_tests/test_dataset_spec.py index 0309aced4..008e8c6fc 100644 --- a/tests/unit/spec_tests/test_dataset_spec.py +++ b/tests/unit/spec_tests/test_dataset_spec.py @@ -245,3 +245,15 @@ def test_data_type_property_value(self): group = GroupSpec('A group', name='group', data_type_inc=data_type_inc, data_type_def=data_type_def) self.assertEqual(group.data_type, data_type) + + def test_build_warn_extra_args(self): + spec_dict = { + 'name': 'dataset1', + 'doc': 'test dataset', + 'dtype': 'int', + 'required': True, + } + msg = ("Unexpected keys ['required'] in spec {'name': 'dataset1', 'doc': 'test dataset', " + "'dtype': 'int', 'required': True}") + with self.assertWarnsWith(UserWarning, msg): + DatasetSpec.build_spec(spec_dict) diff --git a/tests/unit/spec_tests/test_group_spec.py b/tests/unit/spec_tests/test_group_spec.py index 00a937538..31c00cfbb 100644 --- a/tests/unit/spec_tests/test_group_spec.py +++ b/tests/unit/spec_tests/test_group_spec.py @@ -314,6 +314,16 @@ def test_get_namespace_spec(self): expected = AttributeSpec('namespace', 'the namespace for the data type of this object', 'text', required=False) self.assertDictEqual(GroupSpec.get_namespace_spec(), expected) + def test_build_warn_extra_args(self): + spec_dict = { + 'name': 'group1', + 'doc': 'test group', + 'required': True, + } + msg = "Unexpected keys ['required'] in spec {'name': 'group1', 'doc': 'test group', 'required': True}" + with self.assertWarnsWith(UserWarning, msg): + GroupSpec.build_spec(spec_dict) + class TestNotAllowedConfig(TestCase): diff --git a/tests/unit/spec_tests/test_link_spec.py b/tests/unit/spec_tests/test_link_spec.py index e6c680b7c..38e10886b 100644 --- a/tests/unit/spec_tests/test_link_spec.py +++ b/tests/unit/spec_tests/test_link_spec.py @@ -67,3 +67,15 @@ def test_required_is_many(self): ) self.assertEqual(spec.required, req) self.assertEqual(spec.is_many(), many) + + def test_build_warn_extra_args(self): + spec_dict = { + 'name': 'link1', + 'doc': 'test link', + 'target_type': 'TestType', + 'required': True, + } + msg = ("Unexpected keys ['required'] in spec {'name': 'link1', 'doc': 'test link', " + "'target_type': 'TestType', 'required': True}") + with self.assertWarnsWith(UserWarning, msg): + LinkSpec.build_spec(spec_dict)