From 0db5d1cad02ba92753360d6087284a5012d260a2 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 6 May 2024 16:55:40 -0400 Subject: [PATCH 01/12] expose progress bar class control --- src/hdmf/data_utils.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index 23f0b4019..9a14d7863 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -179,6 +179,12 @@ class GenericDataChunkIterator(AbstractDataChunkIterator): doc="Display a progress bar with iteration rate and estimated completion time.", default=False, ), + dict( + name="progress_bar_class", + type=callable, + doc="The progress bar class to use. Defaults to tqdm.tqdm if the TQDM package is installed.", + default=None, + ), dict( name="progress_bar_options", type=None, @@ -277,11 +283,13 @@ def __init__(self, **kwargs): try: from tqdm import tqdm + progress_bar_class = progress_bar_class or tqdm + if "total" in self.progress_bar_options: warn("Option 'total' in 'progress_bar_options' is not allowed to be over-written! Ignoring.") self.progress_bar_options.pop("total") - self.progress_bar = tqdm(total=self.num_buffers, **self.progress_bar_options) + self.progress_bar = progress_bar_class(total=self.num_buffers, **self.progress_bar_options) except ImportError: warn( "You must install tqdm to use the progress bar feature (pip install tqdm)! " From 753522ee2d7d4dfaea172d135b578c327d2dd970 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 15 May 2024 10:03:06 -0400 Subject: [PATCH 02/12] update types --- src/hdmf/data_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index 9a14d7863..ce4f7d5f2 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -181,13 +181,13 @@ class GenericDataChunkIterator(AbstractDataChunkIterator): ), dict( name="progress_bar_class", - type=callable, + type=None, # Would prefer callable, but docval can't handle doc="The progress bar class to use. Defaults to tqdm.tqdm if the TQDM package is installed.", default=None, ), dict( name="progress_bar_options", - type=None, + type=dict, doc="Dictionary of keyword arguments to be passed directly to tqdm.", default=None, ), From c9582cb678e51216c335a259d066d15d66011020 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 15 May 2024 10:08:00 -0400 Subject: [PATCH 03/12] grab progress bar class from kwargs --- src/hdmf/data_utils.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index ce4f7d5f2..68c3f79c9 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -205,8 +205,22 @@ def __init__(self, **kwargs): HDF5 recommends chunk size in the range of 2 to 16 MB for optimal cloud performance. https://youtu.be/rcS5vt-mKok?t=621 """ - buffer_gb, buffer_shape, chunk_mb, chunk_shape, self.display_progress, progress_bar_options = getargs( - "buffer_gb", "buffer_shape", "chunk_mb", "chunk_shape", "display_progress", "progress_bar_options", kwargs + ( + buffer_gb, + buffer_shape, + chunk_mb, + chunk_shape, + self.display_progress, + progress_bar_class, + progress_bar_options, + ) = getargs( + "buffer_gb", + "buffer_shape", + "chunk_mb", + "chunk_shape", + "display_progress", + "progress_bar_options", + kwargs, ) self.progress_bar_options = progress_bar_options or dict() From faa5f52f00a754ef9e4737159d5b6c0a2c0284e6 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 15 May 2024 10:10:08 -0400 Subject: [PATCH 04/12] fix --- src/hdmf/data_utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index 68c3f79c9..69efb357f 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -219,6 +219,7 @@ def __init__(self, **kwargs): "chunk_mb", "chunk_shape", "display_progress", + "progress_bar_class", "progress_bar_options", kwargs, ) From a333784458f6ea60205d88263ff8169b786f7525 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 15 May 2024 14:03:30 -0400 Subject: [PATCH 05/12] swap back to callable but from typing --- src/hdmf/data_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index 69efb357f..5029953f5 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -181,7 +181,7 @@ class GenericDataChunkIterator(AbstractDataChunkIterator): ), dict( name="progress_bar_class", - type=None, # Would prefer callable, but docval can't handle + type=Callable, doc="The progress bar class to use. Defaults to tqdm.tqdm if the TQDM package is installed.", default=None, ), From dd3b3b873096b9d469b214271a29ee89796c1583 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 15 May 2024 14:10:53 -0400 Subject: [PATCH 06/12] swap from typing to collections --- src/hdmf/data_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index 5029953f5..0e83bde2d 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -1,9 +1,9 @@ import copy import math from abc import ABCMeta, abstractmethod -from collections.abc import Iterable +from collections.abc import Iterable, Callable from warnings import warn -from typing import Tuple, Callable +from typing import Tuple from itertools import product, chain import h5py From 2de13f97ea30322202c531abe49ea122aa76c2dc Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 17 May 2024 16:44:03 -0400 Subject: [PATCH 07/12] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47cb8c8d6..e5d820aef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Updated `TermSetWrapper` to support validating a single field within a compound array. @mavaylon1 [#1061](https://github.com/hdmf-dev/hdmf/pull/1061) - Updated testing to not install in editable mode and not run `coverage` by default. @rly [#1107](https://github.com/hdmf-dev/hdmf/pull/1107) - Add `post_init_method` parameter when generating classes to perform post-init functionality, i.e., validation. @mavaylon1 [#1089](https://github.com/hdmf-dev/hdmf/pull/1089) +- Exposed `progress_bar_class` to the `GenericDataChunkIterator` for more custom control over display of progress while iterating. @codycbakerphd [#1110](https://github.com/hdmf-dev/hdmf/pull/1110) ### Bug Fixes - Fixed `TermSetWrapper` warning raised during the setters. @mavaylon1 [#1116](https://github.com/hdmf-dev/hdmf/pull/1116) From d3d77a8761979b3cb9b2467277b6c5f0d00988a0 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 17 May 2024 16:47:28 -0400 Subject: [PATCH 08/12] add test --- .../test_core_GenericDataChunkIterator.py | 27 +++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/unit/utils_test/test_core_GenericDataChunkIterator.py b/tests/unit/utils_test/test_core_GenericDataChunkIterator.py index debac9cab..015cd53aa 100644 --- a/tests/unit/utils_test/test_core_GenericDataChunkIterator.py +++ b/tests/unit/utils_test/test_core_GenericDataChunkIterator.py @@ -408,6 +408,33 @@ def test_progress_bar(self): first_line = file.read() self.assertIn(member=desc, container=first_line) + @unittest.skipIf(not TQDM_INSTALLED, "optional tqdm module is not installed") + def test_progress_bar_class(self): + import tqdm + + def MyCustomProgressBar(tqdm.tqdm): + def update(self, n: int = 1) -> Union[bool, None]: + displayed = super().update(n) + print(f"Custom injection on step {n}") + + return displayed + + out_text_file = self.test_dir / "test_progress_bar_class.txt" + desc = "Testing progress bar..." + with open(file=out_text_file, mode="w") as file: + iterator = self.TestNumpyArrayDataChunkIterator( + array=self.test_array, + display_progress=True, + progress_bar_class=MyCustomProgressBar, + progress_bar_options=dict(file=file, desc=desc), + ) + j = 0 + for buffer in iterator: + j += 1 # dummy operation; must be silent for proper updating of bar + with open(file=out_text_file, mode="r") as file: + first_line = file.read() + self.assertIn(member=desc, container=first_line) + @unittest.skipIf(not TQDM_INSTALLED, "optional tqdm module is installed") def test_progress_bar_no_options(self): dci = self.TestNumpyArrayDataChunkIterator(array=self.test_array, display_progress=True) From 4a3bba6b78d2f7f7961e450cdd36a40ddc6ed3df Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 May 2024 20:47:35 +0000 Subject: [PATCH 09/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .../unit/utils_test/test_core_GenericDataChunkIterator.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/utils_test/test_core_GenericDataChunkIterator.py b/tests/unit/utils_test/test_core_GenericDataChunkIterator.py index 015cd53aa..0dad69e1c 100644 --- a/tests/unit/utils_test/test_core_GenericDataChunkIterator.py +++ b/tests/unit/utils_test/test_core_GenericDataChunkIterator.py @@ -411,14 +411,14 @@ def test_progress_bar(self): @unittest.skipIf(not TQDM_INSTALLED, "optional tqdm module is not installed") def test_progress_bar_class(self): import tqdm - + def MyCustomProgressBar(tqdm.tqdm): def update(self, n: int = 1) -> Union[bool, None]: displayed = super().update(n) print(f"Custom injection on step {n}") - + return displayed - + out_text_file = self.test_dir / "test_progress_bar_class.txt" desc = "Testing progress bar..." with open(file=out_text_file, mode="w") as file: @@ -434,7 +434,7 @@ def update(self, n: int = 1) -> Union[bool, None]: with open(file=out_text_file, mode="r") as file: first_line = file.read() self.assertIn(member=desc, container=first_line) - + @unittest.skipIf(not TQDM_INSTALLED, "optional tqdm module is installed") def test_progress_bar_no_options(self): dci = self.TestNumpyArrayDataChunkIterator(array=self.test_array, display_progress=True) From 83cc64674b82ed832690a105fe1c07426e95c05a Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 17 May 2024 16:48:53 -0400 Subject: [PATCH 10/12] fix --- tests/unit/utils_test/test_core_GenericDataChunkIterator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/utils_test/test_core_GenericDataChunkIterator.py b/tests/unit/utils_test/test_core_GenericDataChunkIterator.py index 0dad69e1c..98b288ca5 100644 --- a/tests/unit/utils_test/test_core_GenericDataChunkIterator.py +++ b/tests/unit/utils_test/test_core_GenericDataChunkIterator.py @@ -412,7 +412,7 @@ def test_progress_bar(self): def test_progress_bar_class(self): import tqdm - def MyCustomProgressBar(tqdm.tqdm): + class MyCustomProgressBar(tqdm.tqdm): def update(self, n: int = 1) -> Union[bool, None]: displayed = super().update(n) print(f"Custom injection on step {n}") From bf71b91c361f995a2b7bf753e82571f208d1c399 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Fri, 17 May 2024 16:50:07 -0400 Subject: [PATCH 11/12] ruff --- tests/unit/utils_test/test_core_GenericDataChunkIterator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/utils_test/test_core_GenericDataChunkIterator.py b/tests/unit/utils_test/test_core_GenericDataChunkIterator.py index 98b288ca5..5dd5da1a5 100644 --- a/tests/unit/utils_test/test_core_GenericDataChunkIterator.py +++ b/tests/unit/utils_test/test_core_GenericDataChunkIterator.py @@ -4,7 +4,7 @@ from pathlib import Path from tempfile import mkdtemp from shutil import rmtree -from typing import Tuple, Iterable, Callable +from typing import Tuple, Iterable, Callable, Union from sys import version_info import h5py @@ -415,7 +415,7 @@ def test_progress_bar_class(self): class MyCustomProgressBar(tqdm.tqdm): def update(self, n: int = 1) -> Union[bool, None]: displayed = super().update(n) - print(f"Custom injection on step {n}") + print(f"Custom injection on step {n}") # noqa: T201 return displayed From 65a300e7b346ffee787c19db4cd1626f99744a71 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 17 May 2024 20:52:01 +0000 Subject: [PATCH 12/12] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- tests/unit/utils_test/test_core_GenericDataChunkIterator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/utils_test/test_core_GenericDataChunkIterator.py b/tests/unit/utils_test/test_core_GenericDataChunkIterator.py index 5dd5da1a5..2117eb6d0 100644 --- a/tests/unit/utils_test/test_core_GenericDataChunkIterator.py +++ b/tests/unit/utils_test/test_core_GenericDataChunkIterator.py @@ -415,7 +415,7 @@ def test_progress_bar_class(self): class MyCustomProgressBar(tqdm.tqdm): def update(self, n: int = 1) -> Union[bool, None]: displayed = super().update(n) - print(f"Custom injection on step {n}") # noqa: T201 + print(f"Custom injection on step {n}") # noqa: T201 return displayed