From 4593adc1d4878642e0a20b83d5264c4c3fb8f380 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:41:45 -0700 Subject: [PATCH 01/12] add condition for multidim string arrays --- src/hdmf/build/objectmapper.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/hdmf/build/objectmapper.py b/src/hdmf/build/objectmapper.py index b5815ee2c..429fd3d7d 100644 --- a/src/hdmf/build/objectmapper.py +++ b/src/hdmf/build/objectmapper.py @@ -617,7 +617,9 @@ def __convert_string(self, value, spec): def string_type(x): return x.isoformat() # method works for both date and datetime if string_type is not None: - if spec.shape is not None or spec.dims is not None: + if spec.shape is not None and len(spec.shape) > 1: + ret = [[string_type(item) for item in sublist] for sublist in value] + elif spec.shape is not None or spec.dims is not None: ret = list(map(string_type, value)) else: ret = string_type(value) From 1577f73e18c27616e75cac9885f2279d3fe17811 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Thu, 15 Aug 2024 10:42:44 -0700 Subject: [PATCH 02/12] add tests for multidim string array build --- tests/unit/build_tests/test_io_map.py | 43 +++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/unit/build_tests/test_io_map.py b/tests/unit/build_tests/test_io_map.py index 63f397682..57d57c725 100644 --- a/tests/unit/build_tests/test_io_map.py +++ b/tests/unit/build_tests/test_io_map.py @@ -9,6 +9,7 @@ from hdmf.testing import TestCase from abc import ABCMeta, abstractmethod import unittest +import numpy as np from tests.unit.helpers.utils import CORE_NAMESPACE, create_test_type_map @@ -353,6 +354,48 @@ def test_build_scalar(self): builder = type_map.build(bar_inst) self.assertEqual(builder.get('data').data, "['a', 'b', 'c', 'd']") + def test_build_2d_lol(self): + bar_spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Bar', + datasets=[ + DatasetSpec( + doc='an example dataset', + dtype='text', + name='data', + shape=(None, None), + attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], + ) + ], + attributes=[AttributeSpec(name='attr1', doc='an example string attribute', dtype='text')], + ) + type_map = self.customSetUp(bar_spec) + type_map.register_map(Bar, BarMapper) + bar_inst = Bar('my_bar', [['aa', 'bb'], ['cc', 'dd']], 'value1', 10) + builder = type_map.build(bar_inst) + self.assertEqual(builder.get('data').data, [['aa', 'bb'], ['cc', 'dd']]) + + def test_build_2d_ndarray(self): + bar_spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Bar', + datasets=[ + DatasetSpec( + doc='an example dataset', + dtype='text', + name='data', + shape=(None, None), + attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], + ) + ], + attributes=[AttributeSpec(name='attr1', doc='an example string attribute', dtype='text')], + ) + type_map = self.customSetUp(bar_spec) + type_map.register_map(Bar, BarMapper) + bar_inst = Bar('my_bar', np.array([['aa', 'bb'], ['cc', 'dd']]), 'value1', 10) + builder = type_map.build(bar_inst) + np.testing.assert_array_equal(builder.get('data').data, np.array([['aa', 'bb'], ['cc', 'dd']])) + def test_build_dataio(self): bar_spec = GroupSpec('A test group specification with a data type', data_type_def='Bar', From 6b95a0a769f1a2746b453c37001aced99ab6dc8e Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Thu, 15 Aug 2024 13:40:33 -0700 Subject: [PATCH 03/12] update condition when defining hdf5 dataset shape --- src/hdmf/backends/hdf5/h5tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/backends/hdf5/h5tools.py b/src/hdmf/backends/hdf5/h5tools.py index 8135d75e7..9d96a0864 100644 --- a/src/hdmf/backends/hdf5/h5tools.py +++ b/src/hdmf/backends/hdf5/h5tools.py @@ -1469,7 +1469,7 @@ def __list_fill__(cls, parent, name, data, options=None): data_shape = io_settings.pop('shape') elif hasattr(data, 'shape'): data_shape = data.shape - elif isinstance(dtype, np.dtype): + elif isinstance(dtype, np.dtype) and len(dtype) > 1: # check if compound dtype data_shape = (len(data),) else: data_shape = get_data_shape(data) From 8555834b2c0219cac194b4223ef6a505949292ed Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Thu, 15 Aug 2024 13:41:54 -0700 Subject: [PATCH 04/12] add test to write multidim string array --- tests/unit/test_io_hdf5_h5tools.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/test_io_hdf5_h5tools.py b/tests/unit/test_io_hdf5_h5tools.py index 5a4fd5a32..1b641f6c6 100644 --- a/tests/unit/test_io_hdf5_h5tools.py +++ b/tests/unit/test_io_hdf5_h5tools.py @@ -164,6 +164,14 @@ def test_write_dataset_list(self): dset = self.f['test_dataset'] self.assertTrue(np.all(dset[:] == a)) + def test_write_dataset_lol_strings(self): + a = [['aa', 'bb'], ['cc', 'dd']] + self.io.write_dataset(self.f, DatasetBuilder('test_dataset', a, attributes={})) + dset = self.f['test_dataset'] + decoded_dset = [[item.decode('utf-8') for item in sublist if isinstance(item, bytes)] + for sublist in dset[:]] + self.assertTrue(decoded_dset == a) + def test_write_dataset_list_compress_gzip(self): a = H5DataIO(np.arange(30).reshape(5, 2, 3), compression='gzip', From c4772d7d07ab9cd699fe5869b54ec7aabab09b46 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Thu, 15 Aug 2024 14:32:11 -0700 Subject: [PATCH 05/12] update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a6369094..d4c3b88d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Enhancements - Added support to append to a dataset of references for HDMF-Zarr. @mavaylon1 [#1157](https://github.com/hdmf-dev/hdmf/pull/1157) +- Added support to write multidimensional string arrays. @stephprince [#1173](https://github.com/hdmf-dev/hdmf/pull/1173) ## HDMF 3.14.3 (July 29, 2024) From d975445cdb324ae343cb1a50acbbec8503332612 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Thu, 15 Aug 2024 14:43:05 -0700 Subject: [PATCH 06/12] fix text decoding in test --- tests/unit/test_io_hdf5_h5tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_io_hdf5_h5tools.py b/tests/unit/test_io_hdf5_h5tools.py index 1b641f6c6..fdfe95e8c 100644 --- a/tests/unit/test_io_hdf5_h5tools.py +++ b/tests/unit/test_io_hdf5_h5tools.py @@ -168,7 +168,7 @@ def test_write_dataset_lol_strings(self): a = [['aa', 'bb'], ['cc', 'dd']] self.io.write_dataset(self.f, DatasetBuilder('test_dataset', a, attributes={})) dset = self.f['test_dataset'] - decoded_dset = [[item.decode('utf-8') for item in sublist if isinstance(item, bytes)] + decoded_dset = [[item.decode('utf-8') if isinstance(item, bytes) else item for item in sublist] for sublist in dset[:]] self.assertTrue(decoded_dset == a) From 5279c9cd33c7477a8e4081136842e3355b5fb86a Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Mon, 19 Aug 2024 15:21:52 -0700 Subject: [PATCH 07/12] add recursive string type for arrays of arbitrary dim --- src/hdmf/build/objectmapper.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/hdmf/build/objectmapper.py b/src/hdmf/build/objectmapper.py index 429fd3d7d..d6e1de15a 100644 --- a/src/hdmf/build/objectmapper.py +++ b/src/hdmf/build/objectmapper.py @@ -598,11 +598,17 @@ def __get_data_type(cls, spec): def __convert_string(self, value, spec): """Convert string types to the specified dtype.""" + def __apply_string_type(value, string_type): + if isinstance(value, (list, tuple, np.ndarray, DataIO)): + return [__apply_string_type(item, string_type) for item in value] + else: + return string_type(value) + ret = value if isinstance(spec, AttributeSpec): if 'text' in spec.dtype: if spec.shape is not None or spec.dims is not None: - ret = list(map(str, value)) + ret = __apply_string_type(value, str) else: ret = str(value) elif isinstance(spec, DatasetSpec): @@ -617,10 +623,8 @@ def __convert_string(self, value, spec): def string_type(x): return x.isoformat() # method works for both date and datetime if string_type is not None: - if spec.shape is not None and len(spec.shape) > 1: - ret = [[string_type(item) for item in sublist] for sublist in value] - elif spec.shape is not None or spec.dims is not None: - ret = list(map(string_type, value)) + if spec.shape is not None or spec.dims is not None: + ret = __apply_string_type(value, string_type) else: ret = string_type(value) # copy over any I/O parameters if they were specified From b636c3fe58550b1d600dc60788883d05ae120194 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Mon, 19 Aug 2024 15:22:44 -0700 Subject: [PATCH 08/12] add test for compound data type with strings --- tests/unit/test_io_hdf5_h5tools.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_io_hdf5_h5tools.py b/tests/unit/test_io_hdf5_h5tools.py index fdfe95e8c..ea51f86d7 100644 --- a/tests/unit/test_io_hdf5_h5tools.py +++ b/tests/unit/test_io_hdf5_h5tools.py @@ -24,7 +24,7 @@ from hdmf.data_utils import DataChunkIterator, GenericDataChunkIterator, InvalidDataIOError from hdmf.spec.catalog import SpecCatalog from hdmf.spec.namespace import NamespaceCatalog, SpecNamespace -from hdmf.spec.spec import GroupSpec +from hdmf.spec.spec import GroupSpec, DtypeSpec from hdmf.testing import TestCase, remove_test_file from hdmf.common.resources import HERD from hdmf.term_set import TermSet, TermSetWrapper @@ -172,6 +172,22 @@ def test_write_dataset_lol_strings(self): for sublist in dset[:]] self.assertTrue(decoded_dset == a) + def test_write_dataset_list_compound_datatype(self): + a = np.array([(1, 2, 0.5), (3, 4, 0.5)], dtype=[('x', 'int'), ('y', 'int'), ('z', 'float')]) + dset_builder = DatasetBuilder( + name='test_dataset', + data=a.tolist(), + attributes={}, + dtype=[ + DtypeSpec('x', doc='x', dtype='int'), + DtypeSpec('y', doc='y', dtype='int'), + DtypeSpec('z', doc='z', dtype='float'), + ], + ) + self.io.write_dataset(self.f, dset_builder) + dset = self.f['test_dataset'] + self.assertTrue(np.all(dset[:] == a)) + def test_write_dataset_list_compress_gzip(self): a = H5DataIO(np.arange(30).reshape(5, 2, 3), compression='gzip', From 4f038d04bdfbcc755ad3dfe3028e82805a64024c Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Mon, 19 Aug 2024 15:37:59 -0700 Subject: [PATCH 09/12] add tests for multidim str attributes --- tests/unit/build_tests/test_classgenerator.py | 7 +- tests/unit/build_tests/test_io_map.py | 84 ++++++++++++++++--- 2 files changed, 76 insertions(+), 15 deletions(-) diff --git a/tests/unit/build_tests/test_classgenerator.py b/tests/unit/build_tests/test_classgenerator.py index 52fdc4839..3c9fda283 100644 --- a/tests/unit/build_tests/test_classgenerator.py +++ b/tests/unit/build_tests/test_classgenerator.py @@ -180,10 +180,11 @@ def test_dynamic_container_creation(self): baz_spec = GroupSpec('A test extension with no Container class', data_type_def='Baz', data_type_inc=self.bar_spec, attributes=[AttributeSpec('attr3', 'a float attribute', 'float'), - AttributeSpec('attr4', 'another float attribute', 'float')]) + AttributeSpec('attr4', 'another float attribute', 'float'), + AttributeSpec('attr_array', 'an array attribute', 'text', shape=(None,)),]) self.spec_catalog.register_spec(baz_spec, 'extension.yaml') cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE) - expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4', 'skip_post_init'} + expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4', 'attr_array', 'skip_post_init'} received_args = set() for x in get_docval(cls.__init__): @@ -211,7 +212,7 @@ def test_dynamic_container_creation_defaults(self): AttributeSpec('attr4', 'another float attribute', 'float')]) self.spec_catalog.register_spec(baz_spec, 'extension.yaml') cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE) - expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4', 'foo', 'skip_post_init'} + expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4', 'attr_array', 'foo', 'skip_post_init'} received_args = set(map(lambda x: x['name'], get_docval(cls.__init__))) self.assertSetEqual(expected_args, received_args) self.assertEqual(cls.__name__, 'Baz') diff --git a/tests/unit/build_tests/test_io_map.py b/tests/unit/build_tests/test_io_map.py index 57d57c725..b2fdb09bb 100644 --- a/tests/unit/build_tests/test_io_map.py +++ b/tests/unit/build_tests/test_io_map.py @@ -21,24 +21,26 @@ class Bar(Container): {'name': 'attr1', 'type': str, 'doc': 'an attribute'}, {'name': 'attr2', 'type': int, 'doc': 'another attribute'}, {'name': 'attr3', 'type': float, 'doc': 'a third attribute', 'default': 3.14}, + {'name': 'attr_array', 'type': 'array_data', 'doc': 'another attribute', 'default': (1, 2, 3)}, {'name': 'foo', 'type': 'Foo', 'doc': 'a group', 'default': None}) def __init__(self, **kwargs): - name, data, attr1, attr2, attr3, foo = getargs('name', 'data', 'attr1', 'attr2', 'attr3', 'foo', kwargs) + name, data, attr1, attr2, attr3, attr_array, foo = getargs('name', 'data', 'attr1', 'attr2', 'attr3', 'attr_array', 'foo', kwargs) super().__init__(name=name) self.__data = data self.__attr1 = attr1 self.__attr2 = attr2 self.__attr3 = attr3 + self.__attr_array = attr_array self.__foo = foo if self.__foo is not None and self.__foo.parent is None: self.__foo.parent = self def __eq__(self, other): - attrs = ('name', 'data', 'attr1', 'attr2', 'attr3', 'foo') + attrs = ('name', 'data', 'attr1', 'attr2', 'attr3', 'attr_array', 'foo') return all(getattr(self, a) == getattr(other, a) for a in attrs) def __str__(self): - attrs = ('name', 'data', 'attr1', 'attr2', 'attr3', 'foo') + attrs = ('name', 'data', 'attr1', 'attr2', 'attr3', 'attr_array', 'foo') return ','.join('%s=%s' % (a, getattr(self, a)) for a in attrs) @property @@ -61,6 +63,10 @@ def attr2(self): def attr3(self): return self.__attr3 + @property + def attr_array(self): + return self.__attr_array + @property def foo(self): return self.__foo @@ -334,12 +340,15 @@ def test_build_1d(self): datasets=[DatasetSpec('an example dataset', 'text', name='data', shape=(None,), attributes=[AttributeSpec( 'attr2', 'an example integer attribute', 'int')])], - attributes=[AttributeSpec('attr1', 'an example string attribute', 'text')]) + attributes=[AttributeSpec('attr1', 'an example string attribute', 'text'), + AttributeSpec('attr_array', 'an example array attribute', 'text', + shape=(None,))]) type_map = self.customSetUp(bar_spec) type_map.register_map(Bar, BarMapper) - bar_inst = Bar('my_bar', ['a', 'b', 'c', 'd'], 'value1', 10) + bar_inst = Bar('my_bar', ['a', 'b', 'c', 'd'], 'value1', 10, attr_array=['a', 'b', 'c', 'd']) builder = type_map.build(bar_inst) - self.assertEqual(builder.get('data').data, ['a', 'b', 'c', 'd']) + np.testing.assert_array_equal(builder.get('data').data, np.array(['a', 'b', 'c', 'd'])) + np.testing.assert_array_equal(builder.get('attr_array'), np.array(['a', 'b', 'c', 'd'])) def test_build_scalar(self): bar_spec = GroupSpec('A test group specification with a data type', @@ -367,13 +376,16 @@ def test_build_2d_lol(self): attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], ) ], - attributes=[AttributeSpec(name='attr1', doc='an example string attribute', dtype='text')], + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', + shape=(None, None))], ) type_map = self.customSetUp(bar_spec) type_map.register_map(Bar, BarMapper) - bar_inst = Bar('my_bar', [['aa', 'bb'], ['cc', 'dd']], 'value1', 10) + str_lol_2d = [['aa', 'bb'], ['cc', 'dd']] + bar_inst = Bar('my_bar', str_lol_2d, 'value1', 10, attr_array=str_lol_2d) builder = type_map.build(bar_inst) - self.assertEqual(builder.get('data').data, [['aa', 'bb'], ['cc', 'dd']]) + self.assertEqual(builder.get('data').data, str_lol_2d) + self.assertEqual(builder.get('attr_array'), str_lol_2d) def test_build_2d_ndarray(self): bar_spec = GroupSpec( @@ -388,13 +400,61 @@ def test_build_2d_ndarray(self): attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], ) ], - attributes=[AttributeSpec(name='attr1', doc='an example string attribute', dtype='text')], + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', shape=(None, None))], + ) + type_map = self.customSetUp(bar_spec) + type_map.register_map(Bar, BarMapper) + str_array_2d = np.array([['aa', 'bb'], ['cc', 'dd']]) + bar_inst = Bar('my_bar', str_array_2d, 'value1', 10, attr_array=str_array_2d) + builder = type_map.build(bar_inst) + np.testing.assert_array_equal(builder.get('data').data, str_array_2d) + np.testing.assert_array_equal(builder.get('attr_array'), str_array_2d) + + def test_build_3d_lol(self): + bar_spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Bar', + datasets=[ + DatasetSpec( + doc='an example dataset', + dtype='text', + name='data', + shape=(None, None, None), + attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], + ) + ], + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', shape=(None, None, None))], + ) + type_map = self.customSetUp(bar_spec) + type_map.register_map(Bar, BarMapper) + str_lol_3d = [[['aa', 'bb'], ['cc', 'dd']], [['ee', 'ff'], ['gg', 'hh']]] + bar_inst = Bar('my_bar', str_lol_3d, 'value1', 10, attr_array=str_lol_3d) + builder = type_map.build(bar_inst) + self.assertEqual(builder.get('data').data, str_lol_3d) + self.assertEqual(builder.get('attr_array'), str_lol_3d) + + def test_build_3d_ndarray(self): + bar_spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Bar', + datasets=[ + DatasetSpec( + doc='an example dataset', + dtype='text', + name='data', + shape=(None, None, None), + attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], + ) + ], + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', shape=(None, None, None))], ) type_map = self.customSetUp(bar_spec) type_map.register_map(Bar, BarMapper) - bar_inst = Bar('my_bar', np.array([['aa', 'bb'], ['cc', 'dd']]), 'value1', 10) + str_array_3d = np.array([[['aa', 'bb'], ['cc', 'dd']], [['ee', 'ff'], ['gg', 'hh']]]) + bar_inst = Bar('my_bar', str_array_3d, 'value1', 10, attr_array=str_array_3d) builder = type_map.build(bar_inst) - np.testing.assert_array_equal(builder.get('data').data, np.array([['aa', 'bb'], ['cc', 'dd']])) + np.testing.assert_array_equal(builder.get('data').data, str_array_3d) + np.testing.assert_array_equal(builder.get('attr_array'), str_array_3d) def test_build_dataio(self): bar_spec = GroupSpec('A test group specification with a data type', From 1ce92aa0f376bb302d1c02f7ad5f8e6f22c787bd Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Mon, 19 Aug 2024 15:39:46 -0700 Subject: [PATCH 10/12] fix line lengths --- tests/unit/build_tests/test_io_map.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/unit/build_tests/test_io_map.py b/tests/unit/build_tests/test_io_map.py index b2fdb09bb..aa4efbb89 100644 --- a/tests/unit/build_tests/test_io_map.py +++ b/tests/unit/build_tests/test_io_map.py @@ -24,7 +24,8 @@ class Bar(Container): {'name': 'attr_array', 'type': 'array_data', 'doc': 'another attribute', 'default': (1, 2, 3)}, {'name': 'foo', 'type': 'Foo', 'doc': 'a group', 'default': None}) def __init__(self, **kwargs): - name, data, attr1, attr2, attr3, attr_array, foo = getargs('name', 'data', 'attr1', 'attr2', 'attr3', 'attr_array', 'foo', kwargs) + name, data, attr1, attr2, attr3, attr_array, foo = getargs('name', 'data', 'attr1', 'attr2', 'attr3', + 'attr_array', 'foo', kwargs) super().__init__(name=name) self.__data = data self.__attr1 = attr1 @@ -400,7 +401,8 @@ def test_build_2d_ndarray(self): attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], ) ], - attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', shape=(None, None))], + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', + shape=(None, None))], ) type_map = self.customSetUp(bar_spec) type_map.register_map(Bar, BarMapper) @@ -423,7 +425,8 @@ def test_build_3d_lol(self): attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], ) ], - attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', shape=(None, None, None))], + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', + shape=(None, None, None))], ) type_map = self.customSetUp(bar_spec) type_map.register_map(Bar, BarMapper) @@ -446,7 +449,8 @@ def test_build_3d_ndarray(self): attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], ) ], - attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', shape=(None, None, None))], + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', + shape=(None, None, None))], ) type_map = self.customSetUp(bar_spec) type_map.register_map(Bar, BarMapper) From a8e5ec76ad6624c27dbff204119b0905359c3dbe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 22:50:45 +0000 Subject: [PATCH 11/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/unit/build_tests/test_io_map.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/build_tests/test_io_map.py b/tests/unit/build_tests/test_io_map.py index aa4efbb89..e095ef318 100644 --- a/tests/unit/build_tests/test_io_map.py +++ b/tests/unit/build_tests/test_io_map.py @@ -24,7 +24,7 @@ class Bar(Container): {'name': 'attr_array', 'type': 'array_data', 'doc': 'another attribute', 'default': (1, 2, 3)}, {'name': 'foo', 'type': 'Foo', 'doc': 'a group', 'default': None}) def __init__(self, **kwargs): - name, data, attr1, attr2, attr3, attr_array, foo = getargs('name', 'data', 'attr1', 'attr2', 'attr3', + name, data, attr1, attr2, attr3, attr_array, foo = getargs('name', 'data', 'attr1', 'attr2', 'attr3', 'attr_array', 'foo', kwargs) super().__init__(name=name) self.__data = data @@ -401,7 +401,7 @@ def test_build_2d_ndarray(self): attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], ) ], - attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', shape=(None, None))], ) type_map = self.customSetUp(bar_spec) @@ -425,7 +425,7 @@ def test_build_3d_lol(self): attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], ) ], - attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', shape=(None, None, None))], ) type_map = self.customSetUp(bar_spec) @@ -449,7 +449,7 @@ def test_build_3d_ndarray(self): attributes=[AttributeSpec(name='attr2', doc='an example integer attribute', dtype='int')], ) ], - attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', + attributes=[AttributeSpec(name='attr_array', doc='an example array attribute', dtype='text', shape=(None, None, None))], ) type_map = self.customSetUp(bar_spec) From dfb9b215c153bd7139c4ade922b6fa7da3e6c2e1 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Mon, 19 Aug 2024 16:06:29 -0700 Subject: [PATCH 12/12] update compound dtype test --- tests/unit/test_io_hdf5_h5tools.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/unit/test_io_hdf5_h5tools.py b/tests/unit/test_io_hdf5_h5tools.py index ea51f86d7..b004a6c54 100644 --- a/tests/unit/test_io_hdf5_h5tools.py +++ b/tests/unit/test_io_hdf5_h5tools.py @@ -186,7 +186,8 @@ def test_write_dataset_list_compound_datatype(self): ) self.io.write_dataset(self.f, dset_builder) dset = self.f['test_dataset'] - self.assertTrue(np.all(dset[:] == a)) + for field in a.dtype.names: + self.assertTrue(np.all(dset[field][:] == a[field])) def test_write_dataset_list_compress_gzip(self): a = H5DataIO(np.arange(30).reshape(5, 2, 3),