From 0dc816a782e202a255eeed8b08792b9cd5d460b3 Mon Sep 17 00:00:00 2001 From: Erik Schomburg Date: Sat, 24 Feb 2024 05:30:20 -0500 Subject: [PATCH 1/5] add ability to send variable plotting options to neuroglancer layers using brackets after name; use for coloring probability vars (#302) Co-authored-by: Erik Schomburg --- chunkflow/flow/neuroglancer.py | 55 ++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 12 deletions(-) diff --git a/chunkflow/flow/neuroglancer.py b/chunkflow/flow/neuroglancer.py index 0acb703..72b062a 100644 --- a/chunkflow/flow/neuroglancer.py +++ b/chunkflow/flow/neuroglancer.py @@ -278,14 +278,26 @@ def _append_segmentation_layer(self, viewer_state: ng.viewer_state.ViewerState, ) ) - def _append_probability_map_layer(self, viewer_state: ng.viewer_state.ViewerState, chunk_name: str, chunk: Chunk): + def _append_probability_map_layer(self, + viewer_state: ng.viewer_state.ViewerState, + chunk_name: str, chunk: Chunk, color=None): if chunk.dtype == np.dtype(' Tuple[str, dict]: + kws = {} + if '[' in varname: + if not varname.endswith(']'): + raise ValueError(f"Unmatched bracket in variable name: '{varname}'") + varname, opts = varname[:-1].split('[') + for arg in opts.split(','): + if '=' in arg: + k, v = arg.split('=') + kws[k] = v + else: + raise ValueError("Only keyword arguments are allowed in neuroglancer variable options") + elif ']' in varname: + raise ValueError(f"Unmatched bracket in variable name: '{varname}'") + + return varname, kws + if selected is None: selected = datas.keys() elif isinstance(selected, str): @@ -335,41 +364,43 @@ def __call__(self, datas: dict, selected: str=None): viewer = ng.Viewer() with viewer.txn() as viewer_state: for name in selected: + name, layer_kwargs = parse_selected_args(name) data = datas[name] + layer_args = (viewer_state, name, data) # breakpoint() if data is None: continue elif isinstance(data, PointCloud): # points - self._append_point_annotation_layer(viewer_state, name, data) + self._append_point_annotation_layer(*layer_args, **layer_kwargs) elif isinstance(data, Synapses): # this could be synapses - self._append_synapse_annotation_layer(viewer_state, name, data) + self._append_synapse_annotation_layer(*layer_args, **layer_kwargs) elif (isinstance(data, defaultdict) or isinstance(data, dict)) \ and len(data)>0: - self._append_skeleton_layer(viewer_state, name, data) + self._append_skeleton_layer(*layer_args, **layer_kwargs) elif isinstance(data, np.ndarray) and 2 == data.ndim and 3 == data.shape[1]: # points - self._append_point_annotation_layer(viewer_state, name, data) + self._append_point_annotation_layer(*layer_args, **layer_kwargs) elif isinstance(data, Chunk): if data.layer_type is None: if data.is_image: - self._append_image_layer(viewer_state, name, data) + self._append_image_layer(*layer_args, **layer_kwargs) elif data.is_segmentation: - self._append_segmentation_layer(viewer_state, name, data) + self._append_segmentation_layer(*layer_args, **layer_kwargs) elif data.is_probability_map: - self._append_probability_map_layer(viewer_state, name, data) + self._append_probability_map_layer(*layer_args, **layer_kwargs) elif data.is_affinity_map: raise ValueError('affinity map is not working yet. To-Do.') else: raise ValueError('unsupported data type.') if data.layer_type == 'segmentation': - self._append_segmentation_layer(viewer_state, name, data) + self._append_segmentation_layer(*layer_args, **layer_kwargs) elif data.layer_type == 'probability_map': - self._append_probability_map_layer(viewer_state, name, data) + self._append_probability_map_layer(*layer_args, **layer_kwargs) elif data.layer_type in set(['image', 'affinity_map']): - self._append_image_layer(viewer_state, name, data) + self._append_image_layer(*layer_args, **layer_kwargs) else: raise ValueError('only support image, affinity map, probability_map, and segmentation for now.') else: From cbd53f2eaf2a41738a2323a47186bbcc4e45e00a Mon Sep 17 00:00:00 2001 From: Erik Schomburg Date: Sat, 24 Feb 2024 05:31:49 -0500 Subject: [PATCH 2/5] fix volume_size param in create-info command for multichannel case (#301) Co-authored-by: Erik Schomburg --- chunkflow/flow/flow.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/chunkflow/flow/flow.py b/chunkflow/flow/flow.py index 154d5ee..201f08d 100755 --- a/chunkflow/flow/flow.py +++ b/chunkflow/flow/flow.py @@ -492,7 +492,7 @@ def cleanup(dir: str, mode: str, suffix: str): type=click.INT, default=0, help = 'maximum mip level.') @operator -def create_info(tasks,input_chunk_name: str, volume_path: str, channel_num: int, +def create_info(tasks, input_chunk_name: str, volume_path: str, channel_num: int, layer_type: str, data_type: str, encoding: str, voxel_size: tuple, voxel_offset: tuple, volume_size: tuple, block_size: tuple, factor: tuple, max_mip: int): """Create attrsdata for Neuroglancer Precomputed volume.""" @@ -506,8 +506,10 @@ def create_info(tasks,input_chunk_name: str, volume_path: str, channel_num: int, chunk = task[input_chunk_name] if chunk.ndim == 3: channel_num = 1 + volume_size = chunk.shape elif chunk.ndim == 4: channel_num = chunk.shape[0] + volume_size = chunk.shape[1:] else: raise ValueError('chunk dimension can only be 3 or 4') @@ -516,7 +518,6 @@ def create_info(tasks,input_chunk_name: str, volume_path: str, channel_num: int, if voxel_size is None: voxel_size = chunk.voxel_size - volume_size = chunk.shape data_type = chunk.dtype.name if layer_type is None: From 97ed0b0d9a65bfa5e19bb7669aea8d8b2a6ae119 Mon Sep 17 00:00:00 2001 From: Erik Schomburg Date: Sat, 24 Feb 2024 05:43:36 -0500 Subject: [PATCH 3/5] Have gaussian_filter plugin operate on copy of input so it does not modify it in-place (#300) * add .idea/ to gitignore * gaussian_filter plugin operates on a copy, so does not modify input * make inplace modification an option in gaussian_filter plugin --------- Co-authored-by: Erik Schomburg --- .gitignore | 1 + chunkflow/plugins/gaussian_filter.py | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index db96116..171c804 100755 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ jwu *.egg-info/ .ipynb_checkpoints/ .vscode/ +*.idea/ docker/inference/build_docker.sh docker/inference/pytorch-emvision/ docker/inference/pytorch-model/ diff --git a/chunkflow/plugins/gaussian_filter.py b/chunkflow/plugins/gaussian_filter.py index cf16839..28d6900 100644 --- a/chunkflow/plugins/gaussian_filter.py +++ b/chunkflow/plugins/gaussian_filter.py @@ -1,9 +1,12 @@ +import numpy as np from chunkflow.chunk import Chunk from scipy.ndimage import gaussian_filter -def execute(chunk: Chunk, sigma: float=1.): +def execute(chunk: Chunk, sigma: float=1., inplace=False): + if not inplace: + chunk = chunk.clone() for z in range(chunk.shape[-3]): if chunk.ndim == 4: for channel in range(chunk.shape[0]): From c183ccb77d56506b7dcbd87fffd4808fc9a68a90 Mon Sep 17 00:00:00 2001 From: Erik Schomburg Date: Sat, 24 Feb 2024 05:47:00 -0500 Subject: [PATCH 4/5] BoundingBox.decompose fix (plus some other minor changes) (#299) * add .idea to gitignore * fix BoundingBox.decompose bug * fix (commented) bug in precomputed volume_path correction * minor style and typing fixes * add BoundingBox.decompose unit test, and additional minor improvements to BBox and Cartesian tests * fix spelling of PhysicalBoudingBox -> PhysicalBoundingBox --------- Co-authored-by: Erik Schomburg --- .gitignore | 1 + chunkflow/chunk/base.py | 6 +++--- chunkflow/flow/save_precomputed.py | 2 +- chunkflow/lib/cartesian_coordinate.py | 26 +++++++++++++------------- chunkflow/volume.py | 8 ++++---- tests/lib/test_cartesian_coordinate.py | 24 ++++++++++++++++++------ 6 files changed, 40 insertions(+), 27 deletions(-) diff --git a/.gitignore b/.gitignore index 171c804..3ff5fa1 100755 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,4 @@ docs/source/_build/ *.DS_Store *.log chunkflow/plugins/chunkflow-plugins +.idea/ diff --git a/chunkflow/chunk/base.py b/chunkflow/chunk/base.py index ea42480..5c47c62 100755 --- a/chunkflow/chunk/base.py +++ b/chunkflow/chunk/base.py @@ -15,7 +15,7 @@ from scipy.ndimage import gaussian_filter from cloudvolume.lib import yellow, Bbox -from chunkflow.lib.cartesian_coordinate import BoundingBox, Cartesian, PhysicalBoudingBox +from chunkflow.lib.cartesian_coordinate import BoundingBox, Cartesian, PhysicalBoundingBox # from typing import Tuple # Offset = Tuple[int, int, int] @@ -568,8 +568,8 @@ def bounding_box(self) -> BoundingBox: return self.bbox @property - def physical_bounding_box(self) -> PhysicalBoudingBox: - return PhysicalBoudingBox( + def physical_bounding_box(self) -> PhysicalBoundingBox: + return PhysicalBoundingBox( self.start, self.stop, self.voxel_size) @property diff --git a/chunkflow/flow/save_precomputed.py b/chunkflow/flow/save_precomputed.py index 16bb061..0cc619e 100644 --- a/chunkflow/flow/save_precomputed.py +++ b/chunkflow/flow/save_precomputed.py @@ -32,7 +32,7 @@ def __init__(self, self.mip = mip # if not volume_path.startswith('precomputed://'): - # volume_path += 'precomputed://' + # volume_path = 'precomputed://' + volume_path self.volume_path = volume_path # gevent.monkey.patch_all(thread=False) diff --git a/chunkflow/lib/cartesian_coordinate.py b/chunkflow/lib/cartesian_coordinate.py index e1f2ee3..d02e666 100644 --- a/chunkflow/lib/cartesian_coordinate.py +++ b/chunkflow/lib/cartesian_coordinate.py @@ -22,13 +22,15 @@ BOUNDING_BOX_RE = re.compile(r'(-?\d+)-(-?\d+)_(-?\d+)-(-?\d+)_(-?\d+)-(-?\d+)(?:\.gz|\.br|\.h5|\.json|\.npy|\.tif|\.csv|\.pkl|\.png|\.jpg)?$') -def to_cartesian(x: Union[tuple, list]): + +def to_cartesian(x: Union[tuple, list, None]): if x is None: return None else: assert len(x) == 3 return Cartesian.from_collection(x) + class Cartesian(namedtuple('Cartesian', ['z', 'y', 'x'])): """Cartesian coordinate or offset.""" __slots__ = () @@ -186,7 +188,7 @@ def inverse(self): @dataclass(frozen=True) -class BoundingBox(): +class BoundingBox: start: Cartesian stop: Cartesian # def __post_init__(self, start, stop) -> BoundingBox: @@ -361,7 +363,7 @@ def __floordiv__(self, other: Number | Cartesian | BoundingBox) -> BoundingBox: minpt = self.minpt // other.minpt maxpt = self.maxpt // other.maxpt elif isinstance(other, np.ndarray): - other = Cartesian.from_collection(other) + other = Cartesian.from_collection(other) minpt = self.start // other maxpt = self.stop // other else: @@ -387,7 +389,6 @@ def __iadd__(self, other: Cartesian | Number): stop = self.stop + other return BoundingBox(start, stop) - def clone(self): return BoundingBox(self.start, self.stop) @@ -468,9 +469,9 @@ def decompose(self, block_size: Cartesian, bboxes = BoundingBoxes() - for z in range(self.start.z, self.stop.z-block_size.z, block_size.z): - for y in range(self.start.y, self.stop.y-block_size.y, block_size.y): - for x in range(self.start.x, self.stop.x-block_size.x, block_size.x): + for z in range(self.start.z, self.stop.z-block_size.z+1, block_size.z): + for y in range(self.start.y, self.stop.y-block_size.y+1, block_size.y): + for x in range(self.start.x, self.stop.x-block_size.x+1, block_size.x): bbox = BoundingBox.from_delta(Cartesian(z,y,x), block_size) bboxes.append(bbox) return bboxes @@ -489,7 +490,7 @@ def shape(self): @cached_property def left_neighbors(self): - sz = self.size3() + sz = self.shape minpt = deepcopy(self.minpt) minpt[0] -= sz[0] @@ -695,12 +696,12 @@ def __len__(self): @dataclass(frozen=True) -class PhysicalBoudingBox(BoundingBox): +class PhysicalBoundingBox(BoundingBox): voxel_size: Cartesian @classmethod def from_bounding_box(cls, bbox: BoundingBox, - voxel_size: Cartesian) -> PhysicalBoudingBox: + voxel_size: Cartesian) -> PhysicalBoundingBox: return cls(bbox.start, bbox.stop, voxel_size) @@ -708,7 +709,7 @@ def from_bounding_box(cls, bbox: BoundingBox, def voxel_bounding_box(self) -> BoundingBox: return BoundingBox(self.start, self.stop) - def to_other_voxel_size(self, voxel_size2: Cartesian) -> PhysicalBoudingBox: + def to_other_voxel_size(self, voxel_size2: Cartesian) -> PhysicalBoundingBox: assert voxel_size2 != self.voxel_size if voxel_size2 >= self.voxel_size: @@ -720,5 +721,4 @@ def to_other_voxel_size(self, voxel_size2: Cartesian) -> PhysicalBoudingBox: factors = self.voxel_size // voxel_size2 start = self.start * factors stop = self.stop * factors - return PhysicalBoudingBox(start, stop, voxel_size2) - + return PhysicalBoundingBox(start, stop, voxel_size2) diff --git a/chunkflow/volume.py b/chunkflow/volume.py index afc0efa..871c047 100644 --- a/chunkflow/volume.py +++ b/chunkflow/volume.py @@ -11,7 +11,7 @@ from cloudvolume import CloudVolume from chunkflow.lib.utils import str_to_dict from .lib.cartesian_coordinate import \ - BoundingBox, Cartesian, BoundingBoxes, PhysicalBoudingBox + BoundingBox, Cartesian, BoundingBoxes, PhysicalBoundingBox from .chunk import Chunk @@ -111,8 +111,8 @@ def block_size(self): self.vol.chunk_size[::-1]) @cached_property - def physical_bounding_box(self) -> PhysicalBoudingBox: - return PhysicalBoudingBox( + def physical_bounding_box(self) -> PhysicalBoundingBox: + return PhysicalBoundingBox( self.start, self.stop, self.voxel_size) @cached_property @@ -296,7 +296,7 @@ def get_candidate_block_bounding_boxes_with_different_voxel_size( voxel_size_low ) for bbox_low in pbbox_low.decompose(block_size_low): - pbbox_low = PhysicalBoudingBox( + pbbox_low = PhysicalBoundingBox( bbox_low.start, bbox_low.stop, voxel_size_low) pbbox_high = pbbox_low.to_other_voxel_size(chunk.voxel_size) chunk_high = block_high.cutout(pbbox_high) diff --git a/tests/lib/test_cartesian_coordinate.py b/tests/lib/test_cartesian_coordinate.py index c5cc231..94eae96 100644 --- a/tests/lib/test_cartesian_coordinate.py +++ b/tests/lib/test_cartesian_coordinate.py @@ -4,11 +4,11 @@ from cloudvolume.lib import Bbox, Vec -from chunkflow.lib.cartesian_coordinate import BoundingBox, Cartesian, to_cartesian, BoundingBoxes, PhysicalBoudingBox +from chunkflow.lib.cartesian_coordinate import BoundingBox, Cartesian, to_cartesian, BoundingBoxes, PhysicalBoundingBox def test_cartesian(): - assert to_cartesian(None) == None + assert to_cartesian(None) is None ct = (1,2,3) assert to_cartesian(ct) == Cartesian(1,2,3) @@ -47,12 +47,13 @@ def test_cartesian(): assert Cartesian(1,2,3).tuple == (1,2,3) assert Cartesian(1,2,3).vec is not None + def test_bounding_box(): bbox = BoundingBox.from_string('3166-3766_7531-8131_2440-3040') - bbox == BoundingBox(Cartesian(3166, 7531, 2440), Cartesian(3766, 8131, 3040)) + assert bbox == BoundingBox(Cartesian(3166, 7531, 2440), Cartesian(3766, 8131, 3040)) bbox = BoundingBox.from_string('Sp1,3166-3766_7531-8131_2440-3040.h5') - bbox == BoundingBox(Cartesian(3166, 7531, 2440), Cartesian(3766, 8131, 3040)) + assert bbox == BoundingBox(Cartesian(3166, 7531, 2440), Cartesian(3766, 8131, 3040)) bbox = Bbox.from_delta((1,3,2), (64, 32, 8)) bbox = BoundingBox.from_bbox(bbox) @@ -65,6 +66,9 @@ def test_bounding_box(): minpt = Cartesian(1,2,3) maxpt = Cartesian(2,3,4) bbox = BoundingBox(minpt, maxpt) + assert bbox.start == minpt + assert bbox.stop == maxpt + assert bbox.shape == Cartesian(1,1,1) bbox = BoundingBox.from_center(Cartesian(1,2,3), 3) assert bbox == BoundingBox.from_list([-2, -1, 0, 4, 5, 6]) @@ -77,6 +81,14 @@ def test_bounding_box(): assert bbox1.union(bbox2) == BoundingBox.from_list([0,1,2, 3,4,5]) assert bbox1.intersection(bbox2) == BoundingBox.from_list([1,2,3, 2,3,4]) + minpt = Cartesian(1,2,3) + maxpt = Cartesian(3,4,5) + bbox = BoundingBox(minpt, maxpt) + bbox_decomp = bbox.decompose(bbox.shape // 2) + assert len(bbox_decomp) == 8 + assert bbox_decomp[0].start == minpt + assert bbox_decomp[-1].stop == maxpt + def test_bounding_boxes(): fname = os.path.join(os.path.dirname(__file__), 'sp3_bboxes.txt') @@ -90,7 +102,7 @@ def test_physical_bounding_box(): start = Cartesian(0, 1, 2) stop = Cartesian(2, 3, 4) voxel_size = Cartesian(2, 2, 2) - pbbox = PhysicalBoudingBox(start, stop, voxel_size) + pbbox = PhysicalBoundingBox(start, stop, voxel_size) pbbox2 = pbbox.to_other_voxel_size(Cartesian(1,1,1)) - assert pbbox2.start == Cartesian(0,2,4) \ No newline at end of file + assert pbbox2.start == Cartesian(0,2,4) From ab12d66e157e2f60928d248770515b9f2e0d605a Mon Sep 17 00:00:00 2001 From: Erik Schomburg Date: Thu, 7 Mar 2024 19:10:43 -0500 Subject: [PATCH 5/5] fix load_png_images to use bounding box starts & stops (#303) * fix load_png_images to use bounding box starts & stops * add conftest.py --------- Co-authored-by: Erik Schomburg --- chunkflow/flow/load_pngs.py | 3 ++- conftest.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) create mode 100644 conftest.py diff --git a/chunkflow/flow/load_pngs.py b/chunkflow/flow/load_pngs.py index cc3e573..0b5f726 100644 --- a/chunkflow/flow/load_pngs.py +++ b/chunkflow/flow/load_pngs.py @@ -16,6 +16,7 @@ def load_image(file_name: str): # img = np.expand_dims(img, axis=0) return img + def load_png_images( path_prefix: str, bbox: BoundingBox = None, @@ -59,7 +60,7 @@ def load_png_images( if os.path.exists(file_name): img = load_image(file_name) img = img.astype(dtype=dtype) - chunk.array[z_offset, :, :] = img + chunk.array[z_offset, :, :] = img[bbox.start[1]:bbox.stop[1], bbox.start[2]:bbox.stop[2]] else: print(f'image file do not exist: {file_name}') diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..0ac4a90 --- /dev/null +++ b/conftest.py @@ -0,0 +1 @@ +pytest_plugins = ['chunkflow']