From 1759f960e179ace17cd49c7de049674ee55c6f13 Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 22 Mar 2023 13:55:03 +0000 Subject: [PATCH 01/26] Initial commit to resolve #6223 --- monai/data/dataset.py | 69 +++++++++++++++---------------------- monai/transforms/compose.py | 60 +++++++++++++++++++++++++++----- 2 files changed, 80 insertions(+), 49 deletions(-) diff --git a/monai/data/dataset.py b/monai/data/dataset.py index 040d583b0d..5e051602ad 100644 --- a/monai/data/dataset.py +++ b/monai/data/dataset.py @@ -316,13 +316,14 @@ def _pre_transform(self, item_transformed): random transform object """ - for _transform in self.transform.transforms: - # execute all the deterministic transforms - if isinstance(_transform, RandomizableTrait) or not isinstance(_transform, Transform): - break - # this is to be consistent with CacheDataset even though it's not in a multi-thread situation. - _xform = deepcopy(_transform) if isinstance(_transform, ThreadUnsafe) else _transform - item_transformed = apply_transform(_xform, item_transformed) + if not isinstance(self.transform, Compose): + raise ValueError("transform must be an instance of monai.transforms.Compose.") + + first_random = self.transform.get_index_of_first( + lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform)) + + item_transformed = self.transform(item_transformed, end=first_random, threading=True) + if self.reset_ops_id: reset_ops_id(item_transformed) return item_transformed @@ -340,16 +341,10 @@ def _post_transform(self, item_transformed): """ if not isinstance(self.transform, Compose): raise ValueError("transform must be an instance of monai.transforms.Compose.") - start_post_randomize_run = False - for _transform in self.transform.transforms: - if ( - start_post_randomize_run - or isinstance(_transform, RandomizableTrait) - or not isinstance(_transform, Transform) - ): - start_post_randomize_run = True - item_transformed = apply_transform(_transform, item_transformed) - return item_transformed + + first_random = self.transform.get_index_of_first( + lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform)) + return self.transform(item_transformed, start=first_random) def _cachecheck(self, item_transformed): """ @@ -492,11 +487,9 @@ def _pre_transform(self, item_transformed): """ if not isinstance(self.transform, Compose): raise ValueError("transform must be an instance of monai.transforms.Compose.") - for i, _transform in enumerate(self.transform.transforms): - if i == self.cache_n_trans: - break - _xform = deepcopy(_transform) if isinstance(_transform, ThreadUnsafe) else _transform - item_transformed = apply_transform(_xform, item_transformed) + + item_transformed = self.transform(item_transformed, end=self.cache_n_trans, threading=True) + reset_ops_id(item_transformed) return item_transformed @@ -512,10 +505,8 @@ def _post_transform(self, item_transformed): """ if not isinstance(self.transform, Compose): raise ValueError("transform must be an instance of monai.transforms.Compose.") - for i, _transform in enumerate(self.transform.transforms): - if i >= self.cache_n_trans: - item_transformed = apply_transform(_transform, item_transformed) - return item_transformed + + return self.transform(item_transformed, start=self.cache_n_trans) class LMDBDataset(PersistentDataset): @@ -879,12 +870,12 @@ def _load_cache_item(self, idx: int): idx: the index of the input data sequence. """ item = self.data[idx] - for _transform in self.transform.transforms: - # execute all the deterministic transforms - if isinstance(_transform, RandomizableTrait) or not isinstance(_transform, Transform): - break - _xform = deepcopy(_transform) if isinstance(_transform, ThreadUnsafe) else _transform - item = apply_transform(_xform, item) + + first_random = self.transform.get_index_of_first( + lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform)) + + item = self.transform(item, end=first_random, threading=True) + if self.as_contiguous: item = convert_to_contiguous(item, memory_format=torch.contiguous_format) return item @@ -911,17 +902,13 @@ def _transform(self, index: int): data = self._cache[cache_index] = self._load_cache_item(cache_index) # load data from cache and execute from the first random transform - start_run = False if not isinstance(self.transform, Compose): raise ValueError("transform must be an instance of monai.transforms.Compose.") - for _transform in self.transform.transforms: - if start_run or isinstance(_transform, RandomizableTrait) or not isinstance(_transform, Transform): - # only need to deep copy data on first non-deterministic transform - if not start_run: - start_run = True - if self.copy_cache: - data = deepcopy(data) - data = apply_transform(_transform, data) + + first_random = self.transform.get_index_of_first( + lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform)) + self.transform(deepcopy(data), start=first_random) + return data diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index 45e706e143..c2f2874093 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -16,11 +16,14 @@ import warnings from collections.abc import Callable, Mapping, Sequence +from copy import deepcopy from typing import Any import numpy as np import monai +from monai.config import NdarrayOrTensor +from monai.transforms.traits import ThreadUnsafe from monai.transforms.inverse import InvertibleTransform # For backwards compatibility (so this still works: from monai.transforms.compose import MapTransform) @@ -152,6 +155,12 @@ def randomize(self, data: Any | None = None) -> None: f'Transform "{tfm_name}" in Compose not randomized\n{tfm_name}.{type_error}.', RuntimeWarning ) + def get_index_of_first(self, predicate): + for i in range(len(self.transforms)): + if predicate(self.transforms[i]): + return i + return None + def flatten(self): """Return a Composition with a simple list of transforms, as opposed to any nested Compositions. @@ -172,11 +181,39 @@ def __len__(self): """Return number of transformations.""" return len(self.flatten().transforms) - def __call__(self, input_): - for _transform in self.transforms: - input_ = apply_transform(_transform, input_, self.map_items, self.unpack_items, self.log_stats) + @classmethod + def execute( + cls, + input_: NdarrayOrTensor, + transforms: Sequence[Any], + map_items: bool = True, + unpack_items: bool = False, + log_stats: bool = False, + start: int = 0, + end: int | None = None, + threading: bool = False + ): + end_ = len(transforms) if end is None else None + if start > end_: + raise ValueError(f"'start' ({start})must be less than 'end' ({end_})") + if end_ > len(transforms): + raise ValueError(f"'end' must be less than or equal to the transform count ({len(transforms)}") + + # no-op if the range is empty + if start == end: + return input_ + + for _transform in transforms[start:end]: + if threading: + _transform = deepcopy(_transform) if isinstance(_transform, ThreadUnsafe) else _transform + input_ = apply_transform(_transform, input_, map_items, unpack_items, log_stats) return input_ + def __call__(self, input_, start=0, end=None, threading=False): + return Compose.execute(input_, self.transforms, + map_items=self.map_items, unpack_items=self.unpack_items, + start=start, end=end, threading=threading) + def inverse(self, data): invertible_transforms = [t for t in self.flatten().transforms if isinstance(t, InvertibleTransform)] if not invertible_transforms: @@ -254,12 +291,17 @@ def flatten(self): weights.append(w) return OneOf(transforms, weights, self.map_items, self.unpack_items) - def __call__(self, data): + def __call__(self, data, start=0, end=None, threading=False): if len(self.transforms) == 0: return data + index = self.R.multinomial(1, self.weights).argmax() _transform = self.transforms[index] - data = apply_transform(_transform, data, self.map_items, self.unpack_items, self.log_stats) + + data = Compose.execute(data, [_transform], + map_items=self.map_items, unpack_items=self.unpack_items, + start=start, end=end, threading=threading) + # if the data is a mapping (dictionary), append the OneOf transform to the end if isinstance(data, monai.data.MetaTensor): self.push_transform(data, extra_info={"index": index}) @@ -318,14 +360,16 @@ def __init__( ) -> None: super().__init__(transforms, map_items, unpack_items, log_stats) - def __call__(self, input_): + def __call__(self, input_, start=0, end=None, threading=False): if len(self.transforms) == 0: return input_ num = len(self.transforms) applied_order = self.R.permutation(range(num)) - for index in applied_order: - input_ = apply_transform(self.transforms[index], input_, self.map_items, self.unpack_items, self.log_stats) + input_ = Compose.execute(input_, [self.transforms[ind] for ind in applied_order], + map_items=self.map_items, unpack_items=self.unpack_items, + start=start, end=end, threading=threading) + # if the data is a mapping (dictionary), append the RandomOrder transform to the end if isinstance(input_, monai.data.MetaTensor): self.push_transform(input_, extra_info={"applied_order": applied_order}) From ba4115d04c3dd6efc040766241cfb4d3bd15043c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 22 Mar 2023 13:57:59 +0000 Subject: [PATCH 02/26] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- monai/data/dataset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/monai/data/dataset.py b/monai/data/dataset.py index 5e051602ad..c3bf8ee617 100644 --- a/monai/data/dataset.py +++ b/monai/data/dataset.py @@ -39,7 +39,6 @@ Compose, Randomizable, RandomizableTrait, - ThreadUnsafe, Transform, apply_transform, convert_to_contiguous, From 632119e28ef0e703af6cde23bf38e2107867a228 Mon Sep 17 00:00:00 2001 From: monai-bot Date: Wed, 22 Mar 2023 14:06:50 +0000 Subject: [PATCH 03/26] [MONAI] code formatting Signed-off-by: monai-bot --- monai/data/dataset.py | 12 +++++--- monai/transforms/compose.py | 58 ++++++++++++++++++++++++------------- 2 files changed, 46 insertions(+), 24 deletions(-) diff --git a/monai/data/dataset.py b/monai/data/dataset.py index c3bf8ee617..b6ea555f28 100644 --- a/monai/data/dataset.py +++ b/monai/data/dataset.py @@ -319,7 +319,8 @@ def _pre_transform(self, item_transformed): raise ValueError("transform must be an instance of monai.transforms.Compose.") first_random = self.transform.get_index_of_first( - lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform)) + lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform) + ) item_transformed = self.transform(item_transformed, end=first_random, threading=True) @@ -342,7 +343,8 @@ def _post_transform(self, item_transformed): raise ValueError("transform must be an instance of monai.transforms.Compose.") first_random = self.transform.get_index_of_first( - lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform)) + lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform) + ) return self.transform(item_transformed, start=first_random) def _cachecheck(self, item_transformed): @@ -871,7 +873,8 @@ def _load_cache_item(self, idx: int): item = self.data[idx] first_random = self.transform.get_index_of_first( - lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform)) + lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform) + ) item = self.transform(item, end=first_random, threading=True) @@ -905,7 +908,8 @@ def _transform(self, index: int): raise ValueError("transform must be an instance of monai.transforms.Compose.") first_random = self.transform.get_index_of_first( - lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform)) + lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform) + ) self.transform(deepcopy(data), start=first_random) return data diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index c2f2874093..e8a3406fa8 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -23,8 +23,8 @@ import monai from monai.config import NdarrayOrTensor -from monai.transforms.traits import ThreadUnsafe from monai.transforms.inverse import InvertibleTransform +from monai.transforms.traits import ThreadUnsafe # For backwards compatibility (so this still works: from monai.transforms.compose import MapTransform) from monai.transforms.transform import ( # noqa: F401 @@ -183,15 +183,15 @@ def __len__(self): @classmethod def execute( - cls, - input_: NdarrayOrTensor, - transforms: Sequence[Any], - map_items: bool = True, - unpack_items: bool = False, - log_stats: bool = False, - start: int = 0, - end: int | None = None, - threading: bool = False + cls, + input_: NdarrayOrTensor, + transforms: Sequence[Any], + map_items: bool = True, + unpack_items: bool = False, + log_stats: bool = False, + start: int = 0, + end: int | None = None, + threading: bool = False, ): end_ = len(transforms) if end is None else None if start > end_: @@ -206,13 +206,19 @@ def execute( for _transform in transforms[start:end]: if threading: _transform = deepcopy(_transform) if isinstance(_transform, ThreadUnsafe) else _transform - input_ = apply_transform(_transform, input_, map_items, unpack_items, log_stats) + input_ = apply_transform(_transform, input_, map_items, unpack_items, log_stats) return input_ def __call__(self, input_, start=0, end=None, threading=False): - return Compose.execute(input_, self.transforms, - map_items=self.map_items, unpack_items=self.unpack_items, - start=start, end=end, threading=threading) + return Compose.execute( + input_, + self.transforms, + map_items=self.map_items, + unpack_items=self.unpack_items, + start=start, + end=end, + threading=threading, + ) def inverse(self, data): invertible_transforms = [t for t in self.flatten().transforms if isinstance(t, InvertibleTransform)] @@ -298,9 +304,15 @@ def __call__(self, data, start=0, end=None, threading=False): index = self.R.multinomial(1, self.weights).argmax() _transform = self.transforms[index] - data = Compose.execute(data, [_transform], - map_items=self.map_items, unpack_items=self.unpack_items, - start=start, end=end, threading=threading) + data = Compose.execute( + data, + [_transform], + map_items=self.map_items, + unpack_items=self.unpack_items, + start=start, + end=end, + threading=threading, + ) # if the data is a mapping (dictionary), append the OneOf transform to the end if isinstance(data, monai.data.MetaTensor): @@ -366,9 +378,15 @@ def __call__(self, input_, start=0, end=None, threading=False): num = len(self.transforms) applied_order = self.R.permutation(range(num)) - input_ = Compose.execute(input_, [self.transforms[ind] for ind in applied_order], - map_items=self.map_items, unpack_items=self.unpack_items, - start=start, end=end, threading=threading) + input_ = Compose.execute( + input_, + [self.transforms[ind] for ind in applied_order], + map_items=self.map_items, + unpack_items=self.unpack_items, + start=start, + end=end, + threading=threading, + ) # if the data is a mapping (dictionary), append the RandomOrder transform to the end if isinstance(input_, monai.data.MetaTensor): From 50a9d06502d1830bc875437b6fabe20af4ec7b6d Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 22 Mar 2023 13:55:03 +0000 Subject: [PATCH 04/26] Initial commit to resolve #6223 Signed-off-by: Ben Murray --- monai/data/dataset.py | 69 +++++++++++++++---------------------- monai/transforms/compose.py | 60 +++++++++++++++++++++++++++----- 2 files changed, 80 insertions(+), 49 deletions(-) diff --git a/monai/data/dataset.py b/monai/data/dataset.py index 040d583b0d..5e051602ad 100644 --- a/monai/data/dataset.py +++ b/monai/data/dataset.py @@ -316,13 +316,14 @@ def _pre_transform(self, item_transformed): random transform object """ - for _transform in self.transform.transforms: - # execute all the deterministic transforms - if isinstance(_transform, RandomizableTrait) or not isinstance(_transform, Transform): - break - # this is to be consistent with CacheDataset even though it's not in a multi-thread situation. - _xform = deepcopy(_transform) if isinstance(_transform, ThreadUnsafe) else _transform - item_transformed = apply_transform(_xform, item_transformed) + if not isinstance(self.transform, Compose): + raise ValueError("transform must be an instance of monai.transforms.Compose.") + + first_random = self.transform.get_index_of_first( + lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform)) + + item_transformed = self.transform(item_transformed, end=first_random, threading=True) + if self.reset_ops_id: reset_ops_id(item_transformed) return item_transformed @@ -340,16 +341,10 @@ def _post_transform(self, item_transformed): """ if not isinstance(self.transform, Compose): raise ValueError("transform must be an instance of monai.transforms.Compose.") - start_post_randomize_run = False - for _transform in self.transform.transforms: - if ( - start_post_randomize_run - or isinstance(_transform, RandomizableTrait) - or not isinstance(_transform, Transform) - ): - start_post_randomize_run = True - item_transformed = apply_transform(_transform, item_transformed) - return item_transformed + + first_random = self.transform.get_index_of_first( + lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform)) + return self.transform(item_transformed, start=first_random) def _cachecheck(self, item_transformed): """ @@ -492,11 +487,9 @@ def _pre_transform(self, item_transformed): """ if not isinstance(self.transform, Compose): raise ValueError("transform must be an instance of monai.transforms.Compose.") - for i, _transform in enumerate(self.transform.transforms): - if i == self.cache_n_trans: - break - _xform = deepcopy(_transform) if isinstance(_transform, ThreadUnsafe) else _transform - item_transformed = apply_transform(_xform, item_transformed) + + item_transformed = self.transform(item_transformed, end=self.cache_n_trans, threading=True) + reset_ops_id(item_transformed) return item_transformed @@ -512,10 +505,8 @@ def _post_transform(self, item_transformed): """ if not isinstance(self.transform, Compose): raise ValueError("transform must be an instance of monai.transforms.Compose.") - for i, _transform in enumerate(self.transform.transforms): - if i >= self.cache_n_trans: - item_transformed = apply_transform(_transform, item_transformed) - return item_transformed + + return self.transform(item_transformed, start=self.cache_n_trans) class LMDBDataset(PersistentDataset): @@ -879,12 +870,12 @@ def _load_cache_item(self, idx: int): idx: the index of the input data sequence. """ item = self.data[idx] - for _transform in self.transform.transforms: - # execute all the deterministic transforms - if isinstance(_transform, RandomizableTrait) or not isinstance(_transform, Transform): - break - _xform = deepcopy(_transform) if isinstance(_transform, ThreadUnsafe) else _transform - item = apply_transform(_xform, item) + + first_random = self.transform.get_index_of_first( + lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform)) + + item = self.transform(item, end=first_random, threading=True) + if self.as_contiguous: item = convert_to_contiguous(item, memory_format=torch.contiguous_format) return item @@ -911,17 +902,13 @@ def _transform(self, index: int): data = self._cache[cache_index] = self._load_cache_item(cache_index) # load data from cache and execute from the first random transform - start_run = False if not isinstance(self.transform, Compose): raise ValueError("transform must be an instance of monai.transforms.Compose.") - for _transform in self.transform.transforms: - if start_run or isinstance(_transform, RandomizableTrait) or not isinstance(_transform, Transform): - # only need to deep copy data on first non-deterministic transform - if not start_run: - start_run = True - if self.copy_cache: - data = deepcopy(data) - data = apply_transform(_transform, data) + + first_random = self.transform.get_index_of_first( + lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform)) + self.transform(deepcopy(data), start=first_random) + return data diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index 45e706e143..c2f2874093 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -16,11 +16,14 @@ import warnings from collections.abc import Callable, Mapping, Sequence +from copy import deepcopy from typing import Any import numpy as np import monai +from monai.config import NdarrayOrTensor +from monai.transforms.traits import ThreadUnsafe from monai.transforms.inverse import InvertibleTransform # For backwards compatibility (so this still works: from monai.transforms.compose import MapTransform) @@ -152,6 +155,12 @@ def randomize(self, data: Any | None = None) -> None: f'Transform "{tfm_name}" in Compose not randomized\n{tfm_name}.{type_error}.', RuntimeWarning ) + def get_index_of_first(self, predicate): + for i in range(len(self.transforms)): + if predicate(self.transforms[i]): + return i + return None + def flatten(self): """Return a Composition with a simple list of transforms, as opposed to any nested Compositions. @@ -172,11 +181,39 @@ def __len__(self): """Return number of transformations.""" return len(self.flatten().transforms) - def __call__(self, input_): - for _transform in self.transforms: - input_ = apply_transform(_transform, input_, self.map_items, self.unpack_items, self.log_stats) + @classmethod + def execute( + cls, + input_: NdarrayOrTensor, + transforms: Sequence[Any], + map_items: bool = True, + unpack_items: bool = False, + log_stats: bool = False, + start: int = 0, + end: int | None = None, + threading: bool = False + ): + end_ = len(transforms) if end is None else None + if start > end_: + raise ValueError(f"'start' ({start})must be less than 'end' ({end_})") + if end_ > len(transforms): + raise ValueError(f"'end' must be less than or equal to the transform count ({len(transforms)}") + + # no-op if the range is empty + if start == end: + return input_ + + for _transform in transforms[start:end]: + if threading: + _transform = deepcopy(_transform) if isinstance(_transform, ThreadUnsafe) else _transform + input_ = apply_transform(_transform, input_, map_items, unpack_items, log_stats) return input_ + def __call__(self, input_, start=0, end=None, threading=False): + return Compose.execute(input_, self.transforms, + map_items=self.map_items, unpack_items=self.unpack_items, + start=start, end=end, threading=threading) + def inverse(self, data): invertible_transforms = [t for t in self.flatten().transforms if isinstance(t, InvertibleTransform)] if not invertible_transforms: @@ -254,12 +291,17 @@ def flatten(self): weights.append(w) return OneOf(transforms, weights, self.map_items, self.unpack_items) - def __call__(self, data): + def __call__(self, data, start=0, end=None, threading=False): if len(self.transforms) == 0: return data + index = self.R.multinomial(1, self.weights).argmax() _transform = self.transforms[index] - data = apply_transform(_transform, data, self.map_items, self.unpack_items, self.log_stats) + + data = Compose.execute(data, [_transform], + map_items=self.map_items, unpack_items=self.unpack_items, + start=start, end=end, threading=threading) + # if the data is a mapping (dictionary), append the OneOf transform to the end if isinstance(data, monai.data.MetaTensor): self.push_transform(data, extra_info={"index": index}) @@ -318,14 +360,16 @@ def __init__( ) -> None: super().__init__(transforms, map_items, unpack_items, log_stats) - def __call__(self, input_): + def __call__(self, input_, start=0, end=None, threading=False): if len(self.transforms) == 0: return input_ num = len(self.transforms) applied_order = self.R.permutation(range(num)) - for index in applied_order: - input_ = apply_transform(self.transforms[index], input_, self.map_items, self.unpack_items, self.log_stats) + input_ = Compose.execute(input_, [self.transforms[ind] for ind in applied_order], + map_items=self.map_items, unpack_items=self.unpack_items, + start=start, end=end, threading=threading) + # if the data is a mapping (dictionary), append the RandomOrder transform to the end if isinstance(input_, monai.data.MetaTensor): self.push_transform(input_, extra_info={"applied_order": applied_order}) From 10edabd27ee06ce1181335b00f09535b00a5bbf2 Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 22 Mar 2023 14:16:11 +0000 Subject: [PATCH 05/26] DCO Remediation Commit for Ben Murray I, Ben Murray , hereby add my Signed-off-by to this commit: 1759f960e179ace17cd49c7de049674ee55c6f13 Signed-off-by: Ben Murray q: --- monai/transforms/compose.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index e8a3406fa8..afcd728499 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -192,7 +192,26 @@ def execute( start: int = 0, end: int | None = None, threading: bool = False, - ): + ) -> NdarrayOrTensor: + """ + ``execute`` provides the implementation that Compose uses to execute a sequence + of transforms. As well as being used by Compose, it can be used by subclasses of + Compose and by code that doesn't have a Compose instance but needs to execute a + sequence of transforms is if it were executed by Compose. For the most part, it + is recommended to use Compose instances, however. + Args: + input_: + transforms: + map_items: + unpack_items: + log_stats: + start: + end: + threading: + + Returns: + + """ end_ = len(transforms) if end is None else None if start > end_: raise ValueError(f"'start' ({start})must be less than 'end' ({end_})") From 5a8ccaf444a1a93efaeaac71cccd0e1776cc831d Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 22 Mar 2023 14:16:11 +0000 Subject: [PATCH 06/26] DCO Remediation Commit for Ben Murray I, Ben Murray , hereby add my Signed-off-by to this commit: 1759f960e179ace17cd49c7de049674ee55c6f13 Signed-off-by: Ben Murray --- monai/transforms/compose.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index e8a3406fa8..afcd728499 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -192,7 +192,26 @@ def execute( start: int = 0, end: int | None = None, threading: bool = False, - ): + ) -> NdarrayOrTensor: + """ + ``execute`` provides the implementation that Compose uses to execute a sequence + of transforms. As well as being used by Compose, it can be used by subclasses of + Compose and by code that doesn't have a Compose instance but needs to execute a + sequence of transforms is if it were executed by Compose. For the most part, it + is recommended to use Compose instances, however. + Args: + input_: + transforms: + map_items: + unpack_items: + log_stats: + start: + end: + threading: + + Returns: + + """ end_ = len(transforms) if end is None else None if start > end_: raise ValueError(f"'start' ({start})must be less than 'end' ({end_})") From c3bd8c09797f236cec0b878da0a1293b0a2a3caa Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 22 Mar 2023 14:20:18 +0000 Subject: [PATCH 07/26] DCO Remediation Commit for Ben Murray I, Ben Murray , hereby add my Signed-off-by to this commit: 10edabd27ee06ce1181335b00f09535b00a5bbf2 Signed-off-by: Ben Murray From 0c0baa443116f3297bd13e37a2a120e5b459931a Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 22 Mar 2023 14:48:07 +0000 Subject: [PATCH 08/26] Fixes to make test_cachedataset, test_persistentdataset and test_cachedataset_persistent_workers pass Signed-off-by: Ben Murray --- monai/data/dataset.py | 7 +++++-- monai/transforms/compose.py | 4 +++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/monai/data/dataset.py b/monai/data/dataset.py index 5dc478c6ba..24f2bb15eb 100644 --- a/monai/data/dataset.py +++ b/monai/data/dataset.py @@ -345,7 +345,9 @@ def _post_transform(self, item_transformed): first_random = self.transform.get_index_of_first( lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform) ) - return self.transform(item_transformed, start=first_random) + if first_random is not None: + item_transformed = self.transform(item_transformed, start=first_random) + return item_transformed def _cachecheck(self, item_transformed): """ @@ -909,7 +911,8 @@ def _transform(self, index: int): first_random = self.transform.get_index_of_first( lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform) ) - self.transform(deepcopy(data), start=first_random) + if first_random is not None: + data = self.transform(deepcopy(data), start=first_random) return data diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index afcd728499..f49a7aa6e7 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -212,7 +212,9 @@ def execute( Returns: """ - end_ = len(transforms) if end is None else None + end_ = len(transforms) if end is None else end + if start is None: + raise ValueError(f"'start' cannot be None") if start > end_: raise ValueError(f"'start' ({start})must be less than 'end' ({end_})") if end_ > len(transforms): From c5a73f6056e5b7a56618e67ee1d8472f6689e7e3 Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 22 Mar 2023 16:21:58 +0000 Subject: [PATCH 09/26] Documentation for Compose.execute Signed-off-by: Ben Murray --- monai/data/dataset.py | 2 +- monai/transforms/compose.py | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/monai/data/dataset.py b/monai/data/dataset.py index 24f2bb15eb..ce03a7dbaa 100644 --- a/monai/data/dataset.py +++ b/monai/data/dataset.py @@ -346,7 +346,7 @@ def _post_transform(self, item_transformed): lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform) ) if first_random is not None: - item_transformed = self.transform(item_transformed, start=first_random) + item_transformed = self.transform(item_transformed, start=first_random) return item_transformed def _cachecheck(self, item_transformed): diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index f49a7aa6e7..d5e081bf92 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -200,11 +200,14 @@ def execute( sequence of transforms is if it were executed by Compose. For the most part, it is recommended to use Compose instances, however. Args: - input_: - transforms: - map_items: - unpack_items: - log_stats: + input_: a tensor-like object to be transformed + transforms: a sequence of transforms to be carried out + map_items: whether to apply the transform to each item in ``data```. + Defaults to True if not set. + unpack_items: whether to unpack parameters using '*'. Defaults to False if not set + log_stats: whether to log detailed information about the application of ``transforms`` + to ``input_``. For NumPy ndarrays and PyTorch tensors, log only the data shape and + value range. Defaults to False if not set. start: end: threading: From 41a156a0295be32f6cdfe3ef7aeaa673fd633cb3 Mon Sep 17 00:00:00 2001 From: Wenqi Li Date: Wed, 22 Mar 2023 17:13:16 +0000 Subject: [PATCH 10/26] style/docs Signed-off-by: Wenqi Li --- monai/transforms/compose.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index d5e081bf92..b0235ae09c 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -200,7 +200,7 @@ def execute( sequence of transforms is if it were executed by Compose. For the most part, it is recommended to use Compose instances, however. Args: - input_: a tensor-like object to be transformed + `input_`: a tensor-like object to be transformed transforms: a sequence of transforms to be carried out map_items: whether to apply the transform to each item in ``data```. Defaults to True if not set. @@ -217,11 +217,11 @@ def execute( """ end_ = len(transforms) if end is None else end if start is None: - raise ValueError(f"'start' cannot be None") + raise ValueError(f"'start' ({start}) cannot be None") if start > end_: - raise ValueError(f"'start' ({start})must be less than 'end' ({end_})") + raise ValueError(f"'start' ({start}) must be less than 'end' ({end_})") if end_ > len(transforms): - raise ValueError(f"'end' must be less than or equal to the transform count ({len(transforms)}") + raise ValueError(f"'end' ({end_}) must be less than or equal to the transform count ({len(transforms)}") # no-op if the range is empty if start == end: @@ -230,7 +230,7 @@ def execute( for _transform in transforms[start:end]: if threading: _transform = deepcopy(_transform) if isinstance(_transform, ThreadUnsafe) else _transform - input_ = apply_transform(_transform, input_, map_items, unpack_items, log_stats) + input_ = apply_transform(_transform, input_, map_items, unpack_items, log_stats) # type: ignore return input_ def __call__(self, input_, start=0, end=None, threading=False): From 61f67e4a0e08766d3a4d8d866ae50f33cdcf3036 Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Fri, 24 Mar 2023 17:03:31 +0000 Subject: [PATCH 11/26] Added tests; updated documentation --- monai/transforms/compose.py | 14 ++-- tests/test_compose.py | 154 +++++++++++++++++++++++++++++++++--- 2 files changed, 149 insertions(+), 19 deletions(-) diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index d5e081bf92..4876670294 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -197,20 +197,22 @@ def execute( ``execute`` provides the implementation that Compose uses to execute a sequence of transforms. As well as being used by Compose, it can be used by subclasses of Compose and by code that doesn't have a Compose instance but needs to execute a - sequence of transforms is if it were executed by Compose. For the most part, it - is recommended to use Compose instances, however. + sequence of transforms is if it were executed by Compose. It should only be used directly + when it is not possible to use ``Compose.__call__`` to achieve the same goal. Args: input_: a tensor-like object to be transformed transforms: a sequence of transforms to be carried out - map_items: whether to apply the transform to each item in ``data```. + map_items: whether to apply the transform to each item in ``data``. Defaults to True if not set. unpack_items: whether to unpack parameters using '*'. Defaults to False if not set log_stats: whether to log detailed information about the application of ``transforms`` to ``input_``. For NumPy ndarrays and PyTorch tensors, log only the data shape and value range. Defaults to False if not set. - start: - end: - threading: + start: the index of the first transform to be executed. If not set, this defaults to 0 + end: the index after the last transform to be exectued. If set, the transform at index-1 + is the last transform that is executed. If this is not set, it defaults to len(transforms) + threading: whether executing is happening in a threaded environment. If set, copies are made + of transforms that have the ``RandomizedTrait`` interface. Returns: diff --git a/tests/test_compose.py b/tests/test_compose.py index ddb7ce25d8..ee61f9bbe1 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -13,9 +13,15 @@ import sys import unittest +from copy import deepcopy + +from parameterized import parameterized + +import numpy as np +import torch from monai.data import DataLoader, Dataset -from monai.transforms import AddChannel, Compose +from monai.transforms import AddChannel, Compose, Flip, Rotate90, Zoom, NormalizeIntensity, Rotate, Rotated from monai.transforms.transform import Randomizable from monai.utils import set_determinism @@ -56,8 +62,12 @@ def b(d): d["b"] += 1 return d - c = Compose([a, b, a, b, a]) - self.assertDictEqual(c({"a": 0, "b": 0}), {"a": 3, "b": 2}) + transforms = [a, b, a, b, a] + data = {"a": 0, "b": 0} + expected = {"a": 3, "b": 2} + + self.assertDictEqual(Compose(transforms)(data), expected) + self.assertDictEqual(Compose.execute(data, transforms), expected) def test_list_dict_compose(self): def a(d): # transform to handle dict data @@ -76,10 +86,15 @@ def c(d): # transform to handle dict data d["c"] += 1 return d - transforms = Compose([a, a, b, c, c]) - value = transforms({"a": 0, "b": 0, "c": 0}) + transforms = [a, a, b, c, c] + data = {"a": 0, "b": 0, "c": 0} + expected = {"a": 2, "b": 1, "c": 2} + value = Compose(transforms)(data) + for item in value: + self.assertDictEqual(item, expected) + value = Compose.execute(data, transforms) for item in value: - self.assertDictEqual(item, {"a": 2, "b": 1, "c": 2}) + self.assertDictEqual(item, expected) def test_non_dict_compose_with_unpack(self): def a(i, i2): @@ -88,8 +103,11 @@ def a(i, i2): def b(i, i2): return i + "b", i2 + "b2" - c = Compose([a, b, a, b], map_items=False, unpack_items=True) - self.assertEqual(c(("", "")), ("abab", "a2b2a2b2")) + transforms = [a, b, a, b] + data = ("", "") + expected = ("abab", "a2b2a2b2") + self.assertEqual(Compose(transforms, map_items=False, unpack_items=True)(data), expected) + self.assertEqual(Compose.execute(data, transforms, map_items=False, unpack_items=True), expected) def test_list_non_dict_compose_with_unpack(self): def a(i, i2): @@ -98,8 +116,11 @@ def a(i, i2): def b(i, i2): return i + "b", i2 + "b2" - c = Compose([a, b, a, b], unpack_items=True) - self.assertEqual(c([("", ""), ("t", "t")]), [("abab", "a2b2a2b2"), ("tabab", "ta2b2a2b2")]) + transforms = [a, b, a, b] + data = [("", ""), ("t", "t")] + expected = [("abab", "a2b2a2b2"), ("tabab", "ta2b2a2b2")] + self.assertEqual(Compose(transforms, unpack_items=True)(data), expected) + self.assertEqual(Compose.execute(data, transforms, unpack_items=True), expected) def test_list_dict_compose_no_map(self): def a(d): # transform to handle dict data @@ -119,10 +140,16 @@ def c(d): # transform to handle dict data di["c"] += 1 return d - transforms = Compose([a, a, b, c, c], map_items=False) - value = transforms({"a": 0, "b": 0, "c": 0}) + transforms = [a, a, b, c, c] + data = {"a": 0, "b": 0, "c": 0} + expected = {"a": 2, "b": 1, "c": 2} + value = Compose(transforms, map_items=False)(data) for item in value: - self.assertDictEqual(item, {"a": 2, "b": 1, "c": 2}) + self.assertDictEqual(item, expected) + value = Compose.execute(data, transforms, map_items=False) + for item in value: + self.assertDictEqual(item, expected) + def test_random_compose(self): class _Acc(Randomizable): @@ -220,5 +247,106 @@ def test_backwards_compatible_imports(self): from monai.transforms.compose import MapTransform, RandomizableTransform, Transform # noqa: F401 +TEST_COMPOSE_EXECUTE_TEST_CASES = [ + [None, tuple()], + [None, (Rotate(np.pi/8),)], + [None, (Flip(0), Flip(1), Rotate90(1), Zoom(0.8), NormalizeIntensity())], + [('a',), (Rotated(('a',), np.pi/8),)], +] + + +class TestComposeExecute(unittest.TestCase): + + @parameterized.expand(TEST_COMPOSE_EXECUTE_TEST_CASES) + def test_compose_execute_equivalence(self, keys, pipeline): + + if keys is None: + data = torch.unsqueeze(torch.tensor(np.arange(24*32).reshape(24, 32)), axis=0) + else: + data = {} + for i_k, k in enumerate(keys): + data[k] = torch.unsqueeze(torch.tensor(np.arange(24*32)).reshape(24, 32) + i_k * 768, + axis=0) + + expected = Compose(deepcopy(pipeline))(data) + + for cutoff in range(len(pipeline)): + + c = Compose(deepcopy(pipeline)) + actual = c(c(data, end=cutoff), start=cutoff) + if isinstance(actual, dict): + for k in actual.keys(): + self.assertTrue(torch.allclose(expected[k], actual[k])) + else: + self.assertTrue(torch.allclose(expected, actual)) + + p = deepcopy(pipeline) + actual = Compose.execute( + Compose.execute(data, p, start=0, end=cutoff), p, start=cutoff) + if isinstance(actual, dict): + for k in actual.keys(): + self.assertTrue(torch.allclose(expected[k], actual[k])) + else: + self.assertTrue(torch.allclose(expected, actual)) + + +class TestOps: + + @staticmethod + def concat(value): + def _inner(data): + return data + value + + return _inner + + @staticmethod + def concatd(value): + def _inner(data): + return {k: v + value for k, v in data.items()} + + return _inner + + @staticmethod + def concata(value): + def _inner(data1, data2): + return data1 + value, data2 + value + + return _inner + + +TEST_COMPOSE_EXECUTE_FLAG_TEST_CASES = [ + [{}, ("",), (TestOps.concat('a'), TestOps.concat('b'))], + [{"unpack_items": True}, ("x", "y"), (TestOps.concat('a'), TestOps.concat('b'))], + [{"map_items": False}, {"x": "1", "y": "2"}, (TestOps.concatd('a'), TestOps.concatd('b'))], + [{"unpack_items": True, "map_items": False}, ("x", "y"), (TestOps.concata('a'), TestOps.concata('b'))], +] + + +class TestComposeExecuteWithFlags(unittest.TestCase): + + @parameterized.expand(TEST_COMPOSE_EXECUTE_FLAG_TEST_CASES) + def test_compose_execute_equivalence_with_flags(self, flags, data, pipeline): + expected = Compose(pipeline, **flags)(data) + + for cutoff in range(len(pipeline)): + + c = Compose(deepcopy(pipeline), **flags) + actual = c(c(data, end=cutoff), start=cutoff) + if isinstance(actual, dict): + for k in actual.keys(): + self.assertEqual(expected[k], actual[k]) + else: + self.assertTrue(expected, actual) + + p = deepcopy(pipeline) + actual = Compose.execute( + Compose.execute(data, p, start=0, end=cutoff, **flags), p, start=cutoff, **flags) + if isinstance(actual, dict): + for k in actual.keys(): + self.assertTrue(expected[k], actual[k]) + else: + self.assertTrue(expected, actual) + + if __name__ == "__main__": unittest.main() From 7f37392e413aec5fcd52ad0424a28c28a90d953a Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Fri, 24 Mar 2023 17:16:39 +0000 Subject: [PATCH 12/26] DCO Remediation Commit for Ben Murray I, Ben Murray , hereby add my Signed-off-by to this commit: 61f67e4a0e08766d3a4d8d866ae50f33cdcf3036 Signed-off-by: Ben Murray From 5301af1dfd04596a782c54dbec6428928664641f Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Fri, 24 Mar 2023 17:36:44 +0000 Subject: [PATCH 13/26] Honoring the self.copy_cache flag Signed-off-by: Ben Murray --- monai/data/dataset.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/monai/data/dataset.py b/monai/data/dataset.py index ce03a7dbaa..c0ea4ef074 100644 --- a/monai/data/dataset.py +++ b/monai/data/dataset.py @@ -912,7 +912,8 @@ def _transform(self, index: int): lambda t: isinstance(t, RandomizableTrait) or not isinstance(t, Transform) ) if first_random is not None: - data = self.transform(deepcopy(data), start=first_random) + data = deepcopy(data) if self.copy_cache is True else data + data = self.transform(data, start=first_random) return data From 6bdedac9f1b36c1d81d0826f9852b01450a2dec0 Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Fri, 24 Mar 2023 18:30:55 +0000 Subject: [PATCH 14/26] Updating for lazy resampling --- monai/transforms/compose.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index ba81d9b7b3..b964eaa918 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -22,6 +22,8 @@ import numpy as np import monai +import monai.transforms as mt +from monai.apps.utils import get_logger from monai.config import NdarrayOrTensor from monai.transforms.inverse import InvertibleTransform from monai.transforms.traits import ThreadUnsafe @@ -288,10 +290,14 @@ def execute( transforms: Sequence[Any], map_items: bool = True, unpack_items: bool = False, - log_stats: bool = False, start: int = 0, end: int | None = None, + lazy_evaluation: bool = False, + overrides: dict = None, + override_keys: tuple = None, threading: bool = False, + log_stats: bool = False, + verbose: bool = False, ) -> NdarrayOrTensor: """ ``execute`` provides the implementation that Compose uses to execute a sequence @@ -332,7 +338,18 @@ def execute( for _transform in transforms[start:end]: if threading: _transform = deepcopy(_transform) if isinstance(_transform, ThreadUnsafe) else _transform - input_ = apply_transform(_transform, input_, map_items, unpack_items, log_stats) # type: ignore + input_ = evaluate_with_overrides(input_, _transform, + lazy_evaluation=lazy_evaluation, + overrides=overrides, + override_keys=override_keys, + verbose=verbose) + input_ = apply_transform(_transform, input_, map_items, unpack_items, log_stats) + input_ = evaluate_with_overrides(input_, None, + lazy_evaluation=lazy_evaluation, + overrides=overrides, + override_keys=override_keys, + verbose=verbose) + return input_ def evaluate_with_overrides(self, input_, upcoming_xform): """ @@ -348,17 +365,21 @@ def evaluate_with_overrides(self, input_, upcoming_xform): override_keys=self.override_keys, verbose=self.verbose, ) - return input_ def __call__(self, input_, start=0, end=None, threading=False): return Compose.execute( input_, self.transforms, - map_items=self.map_items, - unpack_items=self.unpack_items, start=start, end=end, + map_items=self.map_items, + unpack_items=self.unpack_items, + lazy_evaluation=self.lazy_evaluation, + overrides=self.overrides, + override_keys=self.override_keys, threading=threading, + log_stats=self.log_stats, + verbose=self.verbose ) def inverse(self, data): From d99b2b90459152ea45c4576819c2767f43105d2e Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Fri, 24 Mar 2023 20:43:37 +0000 Subject: [PATCH 15/26] Autoformatting Signed-off-by: Ben Murray --- monai/transforms/compose.py | 28 +++++++++++++++++----------- tests/test_compose.py | 35 ++++++++++++----------------------- 2 files changed, 29 insertions(+), 34 deletions(-) diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index b964eaa918..beb4b76314 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -338,17 +338,23 @@ def execute( for _transform in transforms[start:end]: if threading: _transform = deepcopy(_transform) if isinstance(_transform, ThreadUnsafe) else _transform - input_ = evaluate_with_overrides(input_, _transform, - lazy_evaluation=lazy_evaluation, - overrides=overrides, - override_keys=override_keys, - verbose=verbose) + input_ = evaluate_with_overrides( + input_, + _transform, + lazy_evaluation=lazy_evaluation, + overrides=overrides, + override_keys=override_keys, + verbose=verbose, + ) input_ = apply_transform(_transform, input_, map_items, unpack_items, log_stats) - input_ = evaluate_with_overrides(input_, None, - lazy_evaluation=lazy_evaluation, - overrides=overrides, - override_keys=override_keys, - verbose=verbose) + input_ = evaluate_with_overrides( + input_, + None, + lazy_evaluation=lazy_evaluation, + overrides=overrides, + override_keys=override_keys, + verbose=verbose, + ) return input_ def evaluate_with_overrides(self, input_, upcoming_xform): @@ -379,7 +385,7 @@ def __call__(self, input_, start=0, end=None, threading=False): override_keys=self.override_keys, threading=threading, log_stats=self.log_stats, - verbose=self.verbose + verbose=self.verbose, ) def inverse(self, data): diff --git a/tests/test_compose.py b/tests/test_compose.py index ee61f9bbe1..190ef6a18a 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -15,13 +15,12 @@ import unittest from copy import deepcopy -from parameterized import parameterized - import numpy as np import torch +from parameterized import parameterized from monai.data import DataLoader, Dataset -from monai.transforms import AddChannel, Compose, Flip, Rotate90, Zoom, NormalizeIntensity, Rotate, Rotated +from monai.transforms import AddChannel, Compose, Flip, NormalizeIntensity, Rotate, Rotate90, Rotated, Zoom from monai.transforms.transform import Randomizable from monai.utils import set_determinism @@ -150,7 +149,6 @@ def c(d): # transform to handle dict data for item in value: self.assertDictEqual(item, expected) - def test_random_compose(self): class _Acc(Randomizable): self.rand = 0.0 @@ -249,29 +247,25 @@ def test_backwards_compatible_imports(self): TEST_COMPOSE_EXECUTE_TEST_CASES = [ [None, tuple()], - [None, (Rotate(np.pi/8),)], + [None, (Rotate(np.pi / 8),)], [None, (Flip(0), Flip(1), Rotate90(1), Zoom(0.8), NormalizeIntensity())], - [('a',), (Rotated(('a',), np.pi/8),)], + [("a",), (Rotated(("a",), np.pi / 8),)], ] class TestComposeExecute(unittest.TestCase): - @parameterized.expand(TEST_COMPOSE_EXECUTE_TEST_CASES) def test_compose_execute_equivalence(self, keys, pipeline): - if keys is None: - data = torch.unsqueeze(torch.tensor(np.arange(24*32).reshape(24, 32)), axis=0) + data = torch.unsqueeze(torch.tensor(np.arange(24 * 32).reshape(24, 32)), axis=0) else: data = {} for i_k, k in enumerate(keys): - data[k] = torch.unsqueeze(torch.tensor(np.arange(24*32)).reshape(24, 32) + i_k * 768, - axis=0) + data[k] = torch.unsqueeze(torch.tensor(np.arange(24 * 32)).reshape(24, 32) + i_k * 768, axis=0) expected = Compose(deepcopy(pipeline))(data) for cutoff in range(len(pipeline)): - c = Compose(deepcopy(pipeline)) actual = c(c(data, end=cutoff), start=cutoff) if isinstance(actual, dict): @@ -281,8 +275,7 @@ def test_compose_execute_equivalence(self, keys, pipeline): self.assertTrue(torch.allclose(expected, actual)) p = deepcopy(pipeline) - actual = Compose.execute( - Compose.execute(data, p, start=0, end=cutoff), p, start=cutoff) + actual = Compose.execute(Compose.execute(data, p, start=0, end=cutoff), p, start=cutoff) if isinstance(actual, dict): for k in actual.keys(): self.assertTrue(torch.allclose(expected[k], actual[k])) @@ -291,7 +284,6 @@ def test_compose_execute_equivalence(self, keys, pipeline): class TestOps: - @staticmethod def concat(value): def _inner(data): @@ -315,21 +307,19 @@ def _inner(data1, data2): TEST_COMPOSE_EXECUTE_FLAG_TEST_CASES = [ - [{}, ("",), (TestOps.concat('a'), TestOps.concat('b'))], - [{"unpack_items": True}, ("x", "y"), (TestOps.concat('a'), TestOps.concat('b'))], - [{"map_items": False}, {"x": "1", "y": "2"}, (TestOps.concatd('a'), TestOps.concatd('b'))], - [{"unpack_items": True, "map_items": False}, ("x", "y"), (TestOps.concata('a'), TestOps.concata('b'))], + [{}, ("",), (TestOps.concat("a"), TestOps.concat("b"))], + [{"unpack_items": True}, ("x", "y"), (TestOps.concat("a"), TestOps.concat("b"))], + [{"map_items": False}, {"x": "1", "y": "2"}, (TestOps.concatd("a"), TestOps.concatd("b"))], + [{"unpack_items": True, "map_items": False}, ("x", "y"), (TestOps.concata("a"), TestOps.concata("b"))], ] class TestComposeExecuteWithFlags(unittest.TestCase): - @parameterized.expand(TEST_COMPOSE_EXECUTE_FLAG_TEST_CASES) def test_compose_execute_equivalence_with_flags(self, flags, data, pipeline): expected = Compose(pipeline, **flags)(data) for cutoff in range(len(pipeline)): - c = Compose(deepcopy(pipeline), **flags) actual = c(c(data, end=cutoff), start=cutoff) if isinstance(actual, dict): @@ -339,8 +329,7 @@ def test_compose_execute_equivalence_with_flags(self, flags, data, pipeline): self.assertTrue(expected, actual) p = deepcopy(pipeline) - actual = Compose.execute( - Compose.execute(data, p, start=0, end=cutoff, **flags), p, start=cutoff, **flags) + actual = Compose.execute(Compose.execute(data, p, start=0, end=cutoff, **flags), p, start=cutoff, **flags) if isinstance(actual, dict): for k in actual.keys(): self.assertTrue(expected[k], actual[k]) From d93939581ce2ad1cf473044592302c9860370768 Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 29 Mar 2023 10:31:25 +0100 Subject: [PATCH 16/26] Moving Compose.execute to execute_compose as per @ericspod's request. Documenting lazy resampling functionality and updating dataset docs to refer to it as per @wyli's request Signed-off-by: Ben Murray --- monai/data/dataset.py | 11 +- monai/transforms/compose.py | 197 +++++++++++++++++++++--------------- 2 files changed, 127 insertions(+), 81 deletions(-) diff --git a/monai/data/dataset.py b/monai/data/dataset.py index c0ea4ef074..7df19d88d3 100644 --- a/monai/data/dataset.py +++ b/monai/data/dataset.py @@ -208,6 +208,11 @@ class PersistentDataset(Dataset): not guaranteed, so caution should be used when modifying transforms to avoid unexpected errors. If in doubt, it is advisable to clear the cache directory. + Lazy Resampling: + If you make use of the lazy resampling feature of `monai.transforms.Compose`, please refer to + its documentation to familiarize yourself with the interaction between `PersistentDataset` and + lazy resampling. + """ def __init__( @@ -734,6 +739,11 @@ class CacheDataset(Dataset): So to debug or verify the program before real training, users can set `cache_rate=0.0` or `cache_num=0` to temporarily skip caching. + Lazy Resampling: + If you make use of the lazy resampling feature of `monai.transforms.Compose`, please refer to + its documentation to familiarize yourself with the interaction between `CacheDataset` and + lazy resampling. + """ def __init__( @@ -989,7 +999,6 @@ class SmartCacheDataset(Randomizable, CacheDataset): as_contiguous: whether to convert the cached NumPy array or PyTorch tensor to be contiguous. it may help improve the performance of following logic. runtime_cache: Default to `False`, other options are not implemented yet. - """ def __init__( diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index e54f81d6aa..57f64aee3a 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -118,6 +118,79 @@ def evaluate_with_overrides( return data +def execute_compose( + input_: NdarrayOrTensor, + transforms: Sequence[Any], + map_items: bool = True, + unpack_items: bool = False, + start: int = 0, + end: int | None = None, + lazy_evaluation: bool = False, + overrides: dict = None, + override_keys: tuple = None, + threading: bool = False, + log_stats: bool = False, + verbose: bool = False, +) -> NdarrayOrTensor: + """ + ``execute_compose`` provides the implementation that the ``Compose`` class uses to execute a sequence + of transforms. As well as being used by Compose, it can be used by subclasses of + Compose and by code that doesn't have a Compose instance but needs to execute a + sequence of transforms is if it were executed by Compose. It should only be used directly + when it is not possible to use ``Compose.__call__`` to achieve the same goal. + Args: + `input_`: a tensor-like object to be transformed + transforms: a sequence of transforms to be carried out + map_items: whether to apply the transform to each item in ``data``. + Defaults to True if not set. + unpack_items: whether to unpack parameters using '*'. Defaults to False if not set + log_stats: whether to log detailed information about the application of ``transforms`` + to ``input_``. For NumPy ndarrays and PyTorch tensors, log only the data shape and + value range. Defaults to False if not set. + start: the index of the first transform to be executed. If not set, this defaults to 0 + end: the index after the last transform to be exectued. If set, the transform at index-1 + is the last transform that is executed. If this is not set, it defaults to len(transforms) + threading: whether executing is happening in a threaded environment. If set, copies are made + of transforms that have the ``RandomizedTrait`` interface. + + Returns: + + """ + end_ = len(transforms) if end is None else end + if start is None: + raise ValueError(f"'start' ({start}) cannot be None") + if start > end_: + raise ValueError(f"'start' ({start}) must be less than 'end' ({end_})") + if end_ > len(transforms): + raise ValueError(f"'end' ({end_}) must be less than or equal to the transform count ({len(transforms)}") + + # no-op if the range is empty + if start == end: + return input_ + + for _transform in transforms[start:end]: + if threading: + _transform = deepcopy(_transform) if isinstance(_transform, ThreadUnsafe) else _transform + input_ = evaluate_with_overrides( + input_, + _transform, + lazy_evaluation=lazy_evaluation, + overrides=overrides, + override_keys=override_keys, + verbose=verbose, + ) + input_ = apply_transform(_transform, input_, map_items, unpack_items, log_stats) + input_ = evaluate_with_overrides( + input_, + None, + lazy_evaluation=lazy_evaluation, + overrides=overrides, + override_keys=override_keys, + verbose=verbose, + ) + return input_ + + class Compose(Randomizable, InvertibleTransform): """ ``Compose`` provides the ability to chain a series of callables together in @@ -186,6 +259,37 @@ class Compose(Randomizable, InvertibleTransform): calls your pre-processing functions taking into account that not all of them are called on the labels. + Lazy resampling: + Lazy resampling is an experimental feature introduced in 1.2. Its purpose is + to reduce the number of resample operations that must be carried out when executing + a pipeline of transforms. This can provide significant performance improvements in + terms of pipeline executing speed and memory usage, but can also significantly + reduce the loss of information that occurs when performing a number of spatial + resamples in succession. + + Lazy resampling can be thought of as acting in a similar fashion to the `Affine` & `RandAffine` + transforms, in that they allow several spatial transform operations can be specified and carried out with + a single resample step. Unlike these transforms, however, lazy resampling can operate on any set of + transforms specified in any ordering. The user is free to mix monai transforms with transforms from other + libraries; lazy resampling will determine the minimum number of resample steps required in order to + execute the pipeline. + + Lazy resampling works with monai `Dataset` classes that provide caching and persistence. However, if you + are implementing your own caching dataset implementation and wish to make use of lazy resampling, you + should ensure that you fully execute the part of the pipeline that generates the data to be cached + before caching it. This is quite simply done however, as shown by the following example. + + Example: + # run the part of the pipeline that needs to be cached + data = self.transform(data, end=self.post_cache_index) + + # --- + + # fetch the data from the cache and run the rest of the pipeline + data = get_data_from_my_cache(data) + data = self.transform(data, start=self.post_cache_index) + + Args: transforms: sequence of callables. map_items: whether to apply transform to each item in the input `data` if `data` is a list or tuple. @@ -287,80 +391,6 @@ def __len__(self): """Return number of transformations.""" return len(self.flatten().transforms) - @classmethod - def execute( - cls, - input_: NdarrayOrTensor, - transforms: Sequence[Any], - map_items: bool = True, - unpack_items: bool = False, - start: int = 0, - end: int | None = None, - lazy_evaluation: bool = False, - overrides: dict = None, - override_keys: tuple = None, - threading: bool = False, - log_stats: bool = False, - verbose: bool = False, - ) -> NdarrayOrTensor: - """ - ``execute`` provides the implementation that Compose uses to execute a sequence - of transforms. As well as being used by Compose, it can be used by subclasses of - Compose and by code that doesn't have a Compose instance but needs to execute a - sequence of transforms is if it were executed by Compose. It should only be used directly - when it is not possible to use ``Compose.__call__`` to achieve the same goal. - Args: - `input_`: a tensor-like object to be transformed - transforms: a sequence of transforms to be carried out - map_items: whether to apply the transform to each item in ``data``. - Defaults to True if not set. - unpack_items: whether to unpack parameters using '*'. Defaults to False if not set - log_stats: whether to log detailed information about the application of ``transforms`` - to ``input_``. For NumPy ndarrays and PyTorch tensors, log only the data shape and - value range. Defaults to False if not set. - start: the index of the first transform to be executed. If not set, this defaults to 0 - end: the index after the last transform to be exectued. If set, the transform at index-1 - is the last transform that is executed. If this is not set, it defaults to len(transforms) - threading: whether executing is happening in a threaded environment. If set, copies are made - of transforms that have the ``RandomizedTrait`` interface. - - Returns: - - """ - end_ = len(transforms) if end is None else end - if start is None: - raise ValueError(f"'start' ({start}) cannot be None") - if start > end_: - raise ValueError(f"'start' ({start}) must be less than 'end' ({end_})") - if end_ > len(transforms): - raise ValueError(f"'end' ({end_}) must be less than or equal to the transform count ({len(transforms)}") - - # no-op if the range is empty - if start == end: - return input_ - - for _transform in transforms[start:end]: - if threading: - _transform = deepcopy(_transform) if isinstance(_transform, ThreadUnsafe) else _transform - input_ = evaluate_with_overrides( - input_, - _transform, - lazy_evaluation=lazy_evaluation, - overrides=overrides, - override_keys=override_keys, - verbose=verbose, - ) - input_ = apply_transform(_transform, input_, map_items, unpack_items, log_stats) - input_ = evaluate_with_overrides( - input_, - None, - lazy_evaluation=lazy_evaluation, - overrides=overrides, - override_keys=override_keys, - verbose=verbose, - ) - return input_ - def evaluate_with_overrides(self, input_, upcoming_xform): """ Args: @@ -377,7 +407,7 @@ def evaluate_with_overrides(self, input_, upcoming_xform): ) def __call__(self, input_, start=0, end=None, threading=False): - return Compose.execute( + return execute_compose( input_, self.transforms, start=start, @@ -496,7 +526,7 @@ def __call__(self, data, start=0, end=None, threading=False): index = self.R.multinomial(1, self.weights).argmax() _transform = self.transforms[index] - data = Compose.execute( + data = execute_compose( data, [_transform], map_items=self.map_items, @@ -590,7 +620,7 @@ def __call__(self, input_, start=0, end=None, threading=False): num = len(self.transforms) applied_order = self.R.permutation(range(num)) - input_ = Compose.execute( + input_ = execute_compose( input_, [self.transforms[ind] for ind in applied_order], map_items=self.map_items, @@ -729,15 +759,22 @@ def _normalize_probabilities(self, weights): return ensure_tuple(list(weights)) - def __call__(self, data): + def __call__(self, data, start=0, end=None, threading=False): if len(self.transforms) == 0: return data sample_size = self.R.randint(self.min_num_transforms, self.max_num_transforms + 1) applied_order = self.R.choice(len(self.transforms), sample_size, replace=self.replace, p=self.weights).tolist() - for i in applied_order: - data = apply_transform(self.transforms[i], data, self.map_items, self.unpack_items, self.log_stats) + data = execute_compose( + data, + self.transforms[applied_order], + map_items=self.map_items, + unpack_items=self.unpack_items, + start=start, + end=end, + threading=threading, + ) if isinstance(data, monai.data.MetaTensor): self.push_transform(data, extra_info={"applied_order": applied_order}) elif isinstance(data, Mapping): From 4ecf2c318c2c8ec4310a860a5f3774393b9a27d0 Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 29 Mar 2023 10:36:48 +0100 Subject: [PATCH 17/26] Test fix: missed Compose.execute to execute_compose changes Signed-off-by: Ben Murray --- tests/test_compose.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/test_compose.py b/tests/test_compose.py index 190ef6a18a..47869b02aa 100644 --- a/tests/test_compose.py +++ b/tests/test_compose.py @@ -21,6 +21,7 @@ from monai.data import DataLoader, Dataset from monai.transforms import AddChannel, Compose, Flip, NormalizeIntensity, Rotate, Rotate90, Rotated, Zoom +from monai.transforms.compose import execute_compose from monai.transforms.transform import Randomizable from monai.utils import set_determinism @@ -66,7 +67,7 @@ def b(d): expected = {"a": 3, "b": 2} self.assertDictEqual(Compose(transforms)(data), expected) - self.assertDictEqual(Compose.execute(data, transforms), expected) + self.assertDictEqual(execute_compose(data, transforms), expected) def test_list_dict_compose(self): def a(d): # transform to handle dict data @@ -91,7 +92,7 @@ def c(d): # transform to handle dict data value = Compose(transforms)(data) for item in value: self.assertDictEqual(item, expected) - value = Compose.execute(data, transforms) + value = execute_compose(data, transforms) for item in value: self.assertDictEqual(item, expected) @@ -106,7 +107,7 @@ def b(i, i2): data = ("", "") expected = ("abab", "a2b2a2b2") self.assertEqual(Compose(transforms, map_items=False, unpack_items=True)(data), expected) - self.assertEqual(Compose.execute(data, transforms, map_items=False, unpack_items=True), expected) + self.assertEqual(execute_compose(data, transforms, map_items=False, unpack_items=True), expected) def test_list_non_dict_compose_with_unpack(self): def a(i, i2): @@ -119,7 +120,7 @@ def b(i, i2): data = [("", ""), ("t", "t")] expected = [("abab", "a2b2a2b2"), ("tabab", "ta2b2a2b2")] self.assertEqual(Compose(transforms, unpack_items=True)(data), expected) - self.assertEqual(Compose.execute(data, transforms, unpack_items=True), expected) + self.assertEqual(execute_compose(data, transforms, unpack_items=True), expected) def test_list_dict_compose_no_map(self): def a(d): # transform to handle dict data @@ -145,7 +146,7 @@ def c(d): # transform to handle dict data value = Compose(transforms, map_items=False)(data) for item in value: self.assertDictEqual(item, expected) - value = Compose.execute(data, transforms, map_items=False) + value = execute_compose(data, transforms, map_items=False) for item in value: self.assertDictEqual(item, expected) @@ -275,7 +276,7 @@ def test_compose_execute_equivalence(self, keys, pipeline): self.assertTrue(torch.allclose(expected, actual)) p = deepcopy(pipeline) - actual = Compose.execute(Compose.execute(data, p, start=0, end=cutoff), p, start=cutoff) + actual = execute_compose(execute_compose(data, p, start=0, end=cutoff), p, start=cutoff) if isinstance(actual, dict): for k in actual.keys(): self.assertTrue(torch.allclose(expected[k], actual[k])) @@ -329,7 +330,7 @@ def test_compose_execute_equivalence_with_flags(self, flags, data, pipeline): self.assertTrue(expected, actual) p = deepcopy(pipeline) - actual = Compose.execute(Compose.execute(data, p, start=0, end=cutoff, **flags), p, start=cutoff, **flags) + actual = execute_compose(execute_compose(data, p, start=0, end=cutoff, **flags), p, start=cutoff, **flags) if isinstance(actual, dict): for k in actual.keys(): self.assertTrue(expected[k], actual[k]) From 948444fe178be4aa02d7ea67a6248e1c6694bd2e Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 29 Mar 2023 10:41:11 +0100 Subject: [PATCH 18/26] DCO Remediation Commit for Ben Murray I, Ben Murray , hereby add my Signed-off-by to this commit: 6bdedac9f1b36c1d81d0826f9852b01450a2dec0 Signed-off-by: Ben Murray --- monai/transforms/compose.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index 57f64aee3a..4ef9238223 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -181,12 +181,7 @@ def execute_compose( ) input_ = apply_transform(_transform, input_, map_items, unpack_items, log_stats) input_ = evaluate_with_overrides( - input_, - None, - lazy_evaluation=lazy_evaluation, - overrides=overrides, - override_keys=override_keys, - verbose=verbose, + input_, None, lazy_evaluation=lazy_evaluation, overrides=overrides, override_keys=override_keys, verbose=verbose ) return input_ From 9097e07896ea67b875603a0baf6fac2b7acae945 Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 29 Mar 2023 11:13:13 +0100 Subject: [PATCH 19/26] Bug fix for SomeOff; generate list of transforms in execution order through list comprehension Signed-off-by: Ben Murray --- monai/transforms/compose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index 4ef9238223..ccfb8b7b53 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -763,7 +763,7 @@ def __call__(self, data, start=0, end=None, threading=False): data = execute_compose( data, - self.transforms[applied_order], + [self.transforms[a] for a in applied_order], map_items=self.map_items, unpack_items=self.unpack_items, start=start, From d701642cfed8e012e61beadc2262dac27296fd5e Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 29 Mar 2023 11:29:16 +0100 Subject: [PATCH 20/26] Documentation for Compose.get_index_of_first Signed-off-by: Ben Murray --- monai/transforms/compose.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index ccfb8b7b53..3e7b75cdfb 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -361,6 +361,32 @@ def randomize(self, data: Any | None = None) -> None: ) def get_index_of_first(self, predicate): + """ + get_index_of_first takes a ``predicate`` and returns the index of the first transform that + satisfies the predicate (ie. makes the predicate return True). If it is unable to find + a transform that satisfies the ``predicate``, it returns None. + + Note: This is only performed on the transforms directly held by this instance. If this + instance has nested ``Compose`` transforms or other transforms that contain transforms, + it does not iterate into them. + + Example: + c = Compose([Flip(...), Rotate90(...), Zoom(...), RandRotate(...), Resize(...)]) + + print(c.get_index_of_first(lambda t: isinstance(t, RandomTrait))) + >>> 3 + print(c.get_index_of_first(lambda t: isinstance(t, Compose))) + >>> None + + Args: + predicate: a callable that takes a single argument and returns a bool. When called + it is passed a transform from the sequence of transforms contained by this compose + instance. + + Returns: The index of the first transform in the sequence for which ``predicate`` returns + True. None if no transform satisfies the ``predicate`` + + """ for i in range(len(self.transforms)): if predicate(self.transforms[i]): return i From 77e465d7a92241ad4298a744d75bbaf204e1c8eb Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 29 Mar 2023 11:31:16 +0100 Subject: [PATCH 21/26] Slight documentation reformatting for Compose.get_index_of_first Signed-off-by: Ben Murray --- monai/transforms/compose.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index 3e7b75cdfb..df5851ffde 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -366,10 +366,6 @@ def get_index_of_first(self, predicate): satisfies the predicate (ie. makes the predicate return True). If it is unable to find a transform that satisfies the ``predicate``, it returns None. - Note: This is only performed on the transforms directly held by this instance. If this - instance has nested ``Compose`` transforms or other transforms that contain transforms, - it does not iterate into them. - Example: c = Compose([Flip(...), Rotate90(...), Zoom(...), RandRotate(...), Resize(...)]) @@ -378,13 +374,20 @@ def get_index_of_first(self, predicate): print(c.get_index_of_first(lambda t: isinstance(t, Compose))) >>> None + Note: + This is only performed on the transforms directly held by this instance. If this + instance has nested ``Compose`` transforms or other transforms that contain transforms, + it does not iterate into them. + + Args: predicate: a callable that takes a single argument and returns a bool. When called it is passed a transform from the sequence of transforms contained by this compose instance. - Returns: The index of the first transform in the sequence for which ``predicate`` returns - True. None if no transform satisfies the ``predicate`` + Returns: + The index of the first transform in the sequence for which ``predicate`` returns + True. None if no transform satisfies the ``predicate`` """ for i in range(len(self.transforms)): From 5605e71f58bd67635a11e4c1f1794f2392392930 Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 29 Mar 2023 12:10:32 +0100 Subject: [PATCH 22/26] Updated docstrings for execute_compose. Renamed input_ to data for execute_compose. Signed-off-by: Ben Murray --- monai/transforms/compose.py | 59 ++++++++++++++++++++++++------------- 1 file changed, 38 insertions(+), 21 deletions(-) diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index df5851ffde..aabb59259f 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -119,19 +119,19 @@ def evaluate_with_overrides( def execute_compose( - input_: NdarrayOrTensor, + data: NdarrayOrTensor | Sequence[NdarrayOrTensor] | Mapping[Any, NdarrayOrTensor], transforms: Sequence[Any], map_items: bool = True, unpack_items: bool = False, start: int = 0, end: int | None = None, lazy_evaluation: bool = False, - overrides: dict = None, - override_keys: tuple = None, + overrides: dict | None = None, + override_keys: tuple | None = None, threading: bool = False, log_stats: bool = False, verbose: bool = False, -) -> NdarrayOrTensor: +) -> NdarrayOrTensor | Sequence[NdarrayOrTensor] | Mapping[Any, NdarrayOrTensor]: """ ``execute_compose`` provides the implementation that the ``Compose`` class uses to execute a sequence of transforms. As well as being used by Compose, it can be used by subclasses of @@ -139,22 +139,39 @@ def execute_compose( sequence of transforms is if it were executed by Compose. It should only be used directly when it is not possible to use ``Compose.__call__`` to achieve the same goal. Args: - `input_`: a tensor-like object to be transformed + data: a tensor-like object to be transformed transforms: a sequence of transforms to be carried out - map_items: whether to apply the transform to each item in ``data``. - Defaults to True if not set. - unpack_items: whether to unpack parameters using '*'. Defaults to False if not set - log_stats: whether to log detailed information about the application of ``transforms`` - to ``input_``. For NumPy ndarrays and PyTorch tensors, log only the data shape and - value range. Defaults to False if not set. + map_items: whether to apply transform to each item in the input `data` if `data` is a list or tuple. + defaults to `True`. + unpack_items: whether to unpack input `data` with `*` as parameters for the callable function of transform. + defaults to `False`. start: the index of the first transform to be executed. If not set, this defaults to 0 end: the index after the last transform to be exectued. If set, the transform at index-1 - is the last transform that is executed. If this is not set, it defaults to len(transforms) + is the last transform that is executed. If this is not set, it defaults to len(transforms) + lazy_evaluation: whether to enable lazy evaluation for lazy transforms. If False, transforms will be + carried out on a transform by transform basis. If True, all lazy transforms will + be executed by accumulating changes and resampling as few times as possible. + A `monai.transforms.Identity[D]` transform in the pipeline will trigger the evaluation of + the pending operations and make the primary data up-to-date. + overrides: this optional parameter allows you to specify a dictionary of parameters that should be overridden + when executing a pipeline. These each parameter that is compatible with a given transform is then applied + to that transform before it is executed. Note that overrides are currently only applied when lazy_evaluation + is True. If lazy_evaluation is False they are ignored. + currently supported args are: + {``"mode"``, ``"padding_mode"``, ``"dtype"``, ``"align_corners"``, ``"resample_mode"``, ``device``}, + please see also :py:func:`monai.transforms.lazy.apply_transforms` for more details. + override_keys: this optional parameter specifies the keys to which ``overrides`` are to be applied. If + ``overrides`` is set, ``override_keys`` must also be set. threading: whether executing is happening in a threaded environment. If set, copies are made - of transforms that have the ``RandomizedTrait`` interface. + of transforms that have the ``RandomizedTrait`` interface. + log_stats: whether to log the detailed information of data and applied transform when error happened, + for NumPy array and PyTorch Tensor, log the data shape and value range, + for other metadata, log the values directly. default to `False`. + verbose: whether to print debugging info when lazy_evaluation=True. Returns: - + A tensorlike, sequence of tensorlikes or dict of tensorlists containing the result of running + `data`` through the sequence of ``transforms``. """ end_ = len(transforms) if end is None else end if start is None: @@ -166,24 +183,24 @@ def execute_compose( # no-op if the range is empty if start == end: - return input_ + return data for _transform in transforms[start:end]: if threading: _transform = deepcopy(_transform) if isinstance(_transform, ThreadUnsafe) else _transform - input_ = evaluate_with_overrides( - input_, + data = evaluate_with_overrides( + data, _transform, lazy_evaluation=lazy_evaluation, overrides=overrides, override_keys=override_keys, verbose=verbose, ) - input_ = apply_transform(_transform, input_, map_items, unpack_items, log_stats) - input_ = evaluate_with_overrides( - input_, None, lazy_evaluation=lazy_evaluation, overrides=overrides, override_keys=override_keys, verbose=verbose + data = apply_transform(_transform, data, map_items, unpack_items, log_stats) + data = evaluate_with_overrides( + data, None, lazy_evaluation=lazy_evaluation, overrides=overrides, override_keys=override_keys, verbose=verbose ) - return input_ + return data class Compose(Randomizable, InvertibleTransform): From c06e014d34eee88aec7a3dcddb43d33ceb5003a5 Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 29 Mar 2023 13:42:30 +0100 Subject: [PATCH 23/26] Fixing errors reported by flake8-py3 (mypy) output Signed-off-by: Ben Murray --- monai/transforms/compose.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index aabb59259f..a0b06bf4c3 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -127,7 +127,7 @@ def execute_compose( end: int | None = None, lazy_evaluation: bool = False, overrides: dict | None = None, - override_keys: tuple | None = None, + override_keys: Sequence[str] | None = None, threading: bool = False, log_stats: bool = False, verbose: bool = False, @@ -334,7 +334,7 @@ def __init__( map_items: bool = True, unpack_items: bool = False, log_stats: bool = False, - lazy_evaluation: bool | None = None, + lazy_evaluation: bool = False, overrides: dict | None = None, override_keys: Sequence[str] | None = None, verbose: bool = False, From 191506d35cb2ebc31cbb8f499097bb4a481db470 Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 29 Mar 2023 14:43:47 +0100 Subject: [PATCH 24/26] Had to go back to lazy_evaluation default of None for now but this is a critical design flaw that needs fixing Signed-off-by: Ben Murray --- monai/transforms/compose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index a0b06bf4c3..be1fcd8b4a 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -334,7 +334,7 @@ def __init__( map_items: bool = True, unpack_items: bool = False, log_stats: bool = False, - lazy_evaluation: bool = False, + lazy_evaluation: bool | None = None, overrides: dict | None = None, override_keys: Sequence[str] | None = None, verbose: bool = False, From ec7140215aaefa10586903926484bc7d809924ee Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 29 Mar 2023 15:04:15 +0100 Subject: [PATCH 25/26] execute_compose type ignore as it can't be fixed without polluting more code with an incorrect type Signed-off-by: Ben Murray --- monai/transforms/compose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index be1fcd8b4a..962ec94fc9 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -461,7 +461,7 @@ def __call__(self, input_, start=0, end=None, threading=False): threading=threading, log_stats=self.log_stats, verbose=self.verbose, - ) + ) # type: ignore def inverse(self, data): invertible_transforms = [t for t in self.flatten().transforms if isinstance(t, InvertibleTransform)] From 85b18de8a093e4582a3cb37a420f9c8585ab80e5 Mon Sep 17 00:00:00 2001 From: Ben Murray Date: Wed, 29 Mar 2023 17:28:07 +0100 Subject: [PATCH 26/26] type: ignore suppression as this is being addressed separately Signed-off-by: Ben Murray --- monai/transforms/compose.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monai/transforms/compose.py b/monai/transforms/compose.py index 962ec94fc9..8a8518c92b 100644 --- a/monai/transforms/compose.py +++ b/monai/transforms/compose.py @@ -455,7 +455,7 @@ def __call__(self, input_, start=0, end=None, threading=False): end=end, map_items=self.map_items, unpack_items=self.unpack_items, - lazy_evaluation=self.lazy_evaluation, + lazy_evaluation=self.lazy_evaluation, # type: ignore overrides=self.overrides, override_keys=self.override_keys, threading=threading,