Merge branch 'master' of github.com:seung-lab/chunkflow

.gitignore

@@ -15,6 +15,7 @@ jwu
 *.egg-info/
 .ipynb_checkpoints/
 .vscode/
+*.idea/
 docker/inference/build_docker.sh
 docker/inference/pytorch-emvision/
 docker/inference/pytorch-model/
@@ -29,3 +30,4 @@ docs/source/_build/
 *.DS_Store
 *.log
 chunkflow/plugins/chunkflow-plugins
+.idea/

chunkflow/chunk/base.py

@@ -15,7 +15,7 @@
 from scipy.ndimage import gaussian_filter
 
 from cloudvolume.lib import yellow, Bbox
-from chunkflow.lib.cartesian_coordinate import BoundingBox, Cartesian, PhysicalBoudingBox
+from chunkflow.lib.cartesian_coordinate import BoundingBox, Cartesian, PhysicalBoundingBox
 
 # from typing import Tuple
 # Offset = Tuple[int, int, int]
@@ -576,8 +576,8 @@ def bounding_box(self) -> BoundingBox:
         return self.bbox
 
     @property
-    def physical_bounding_box(self) -> PhysicalBoudingBox:
-        return PhysicalBoudingBox(
+    def physical_bounding_box(self) -> PhysicalBoundingBox:
+        return PhysicalBoundingBox(
             self.start, self.stop, self.voxel_size)
 
     @property

chunkflow/flow/flow.py

@@ -492,7 +492,7 @@ def cleanup(dir: str, mode: str, suffix: str):
               type=click.INT, default=0, 
               help = 'maximum mip level.')
 @operator
-def create_info(tasks,input_chunk_name: str, volume_path: str, channel_num: int, 
+def create_info(tasks, input_chunk_name: str, volume_path: str, channel_num: int,
                 layer_type: str, data_type: str, encoding: str, voxel_size: tuple, 
                 voxel_offset: tuple, volume_size: tuple, block_size: tuple, factor: tuple, max_mip: int):
     """Create attrsdata for Neuroglancer Precomputed volume."""
@@ -506,8 +506,10 @@ def create_info(tasks,input_chunk_name: str, volume_path: str, channel_num: int,
                 chunk = task[input_chunk_name]
                 if chunk.ndim == 3:
                     channel_num = 1
+                    volume_size = chunk.shape
                 elif chunk.ndim == 4:
                     channel_num = chunk.shape[0]
+                    volume_size = chunk.shape[1:]
                 else:
                     raise ValueError('chunk dimension can only be 3 or 4')
 
@@ -516,7 +518,6 @@ def create_info(tasks,input_chunk_name: str, volume_path: str, channel_num: int,
                 if voxel_size is None:
                     voxel_size = chunk.voxel_size
 
-                volume_size = chunk.shape
                 data_type = chunk.dtype.name
 
                 if layer_type is None:

chunkflow/flow/load_pngs.py

@@ -16,6 +16,7 @@ def load_image(file_name: str):
     # img = np.expand_dims(img, axis=0)
     return img
 
+
 def load_png_images(
         path_prefix: str, 
         bbox: BoundingBox = None, 
@@ -59,7 +60,7 @@ def load_png_images(
         if os.path.exists(file_name):
             img = load_image(file_name)
             img = img.astype(dtype=dtype)
-            chunk.array[z_offset, :, :] = img
+            chunk.array[z_offset, :, :] = img[bbox.start[1]:bbox.stop[1], bbox.start[2]:bbox.stop[2]]
         else:
             print(f'image file do not exist: {file_name}')
 

chunkflow/flow/neuroglancer.py

@@ -278,14 +278,26 @@ def _append_segmentation_layer(self, viewer_state: ng.viewer_state.ViewerState,
             )
         ) 
 
-    def _append_probability_map_layer(self, viewer_state: ng.viewer_state.ViewerState, chunk_name: str, chunk: Chunk):
+    def _append_probability_map_layer(self,
+            viewer_state: ng.viewer_state.ViewerState,
+            chunk_name: str, chunk: Chunk, color=None):
         if chunk.dtype == np.dtype('<f4') or chunk.dtype == np.dtype('float16'):
             chunk = chunk.astype(np.float32)
 
         voxel_size = self._get_voxel_size(chunk)
         # chunk = np.ascontiguousarray(chunk)
         if chunk.shape[0] == 1:
-            shader = """void main() {
+            if color is not None:
+                shader = """#uicontrol vec3 color color(default="%s")
+#uicontrol float brightness slider(min=-1, max=1)
+#uicontrol float contrast slider(min=-3, max=3, step=0.01)
+void main() {
+  emitRGB(color *
+          (toNormalized(getDataValue(0)) + brightness) * exp(contrast));
+}
+""" % color
+            else:
+                shader = """void main() {
 emitGrayscale(toNormalized(getDataValue(0)));
 }
 """ 
@@ -324,6 +336,23 @@ def __call__(self, datas: dict, selected: str=None):
         Parameters:
         chunks: multiple chunks
         """
+        def parse_selected_args(varname: str) -> Tuple[str, dict]:
+            kws = {}
+            if '[' in varname:
+                if not varname.endswith(']'):
+                    raise ValueError(f"Unmatched bracket in variable name: '{varname}'")
+                varname, opts = varname[:-1].split('[')
+                for arg in opts.split(','):
+                    if '=' in arg:
+                        k, v = arg.split('=')
+                        kws[k] = v
+                    else:
+                        raise ValueError("Only keyword arguments are allowed in neuroglancer variable options")
+            elif ']' in varname:
+                raise ValueError(f"Unmatched bracket in variable name: '{varname}'")
+
+            return varname, kws
+
         if selected is None:
             selected = datas.keys()
         elif isinstance(selected, str):
@@ -335,41 +364,43 @@ def __call__(self, datas: dict, selected: str=None):
         viewer = ng.Viewer()
         with viewer.txn() as viewer_state:
             for name in selected:
+                name, layer_kwargs = parse_selected_args(name)
                 data = datas[name]
+                layer_args = (viewer_state, name, data)
                 # breakpoint()
 
                 if data is None:
                     continue
                 elif isinstance(data, PointCloud):
                     # points
-                    self._append_point_annotation_layer(viewer_state, name, data)
+                    self._append_point_annotation_layer(*layer_args, **layer_kwargs)
                 elif isinstance(data, Synapses):
                     # this could be synapses
-                    self._append_synapse_annotation_layer(viewer_state, name, data)
+                    self._append_synapse_annotation_layer(*layer_args, **layer_kwargs)
                 elif (isinstance(data, defaultdict) or isinstance(data, dict)) \
                         and len(data)>0:
-                    self._append_skeleton_layer(viewer_state, name, data)
+                    self._append_skeleton_layer(*layer_args, **layer_kwargs)
                 elif isinstance(data, np.ndarray) and 2 == data.ndim and 3 == data.shape[1]:
                     # points
-                    self._append_point_annotation_layer(viewer_state, name, data)
+                    self._append_point_annotation_layer(*layer_args, **layer_kwargs)
                 elif isinstance(data, Chunk):
                     if data.layer_type is None:
                         if data.is_image:
-                            self._append_image_layer(viewer_state, name, data)
+                            self._append_image_layer(*layer_args, **layer_kwargs)
                         elif data.is_segmentation:
-                            self._append_segmentation_layer(viewer_state, name, data)
+                            self._append_segmentation_layer(*layer_args, **layer_kwargs)
                         elif data.is_probability_map:
-                            self._append_probability_map_layer(viewer_state, name, data)
+                            self._append_probability_map_layer(*layer_args, **layer_kwargs)
                         elif data.is_affinity_map:
                             raise ValueError('affinity map is not working yet. To-Do.')
                         else:
                             raise ValueError('unsupported data type.')
                     if data.layer_type == 'segmentation':
-                        self._append_segmentation_layer(viewer_state, name, data)
+                        self._append_segmentation_layer(*layer_args, **layer_kwargs)
                     elif data.layer_type == 'probability_map':
-                        self._append_probability_map_layer(viewer_state, name, data)
+                        self._append_probability_map_layer(*layer_args, **layer_kwargs)
                     elif data.layer_type in set(['image', 'affinity_map']):
-                        self._append_image_layer(viewer_state, name, data)
+                        self._append_image_layer(*layer_args, **layer_kwargs)
                     else: 
                         raise ValueError('only support image, affinity map, probability_map, and segmentation for now.')
                 else:

chunkflow/flow/save_precomputed.py

@@ -32,7 +32,7 @@ def __init__(self,
         self.mip = mip
 
         # if not volume_path.startswith('precomputed://'):
-        #     volume_path += 'precomputed://'
+        #     volume_path = 'precomputed://' + volume_path
         self.volume_path = volume_path
 
         # gevent.monkey.patch_all(thread=False)

chunkflow/lib/cartesian_coordinate.py

@@ -22,13 +22,15 @@
 
 BOUNDING_BOX_RE = re.compile(r'(-?\d+)-(-?\d+)_(-?\d+)-(-?\d+)_(-?\d+)-(-?\d+)(?:\.gz|\.br|\.h5|\.json|\.npy|\.tif|\.csv|\.pkl|\.png|\.jpg)?$')
 
-def to_cartesian(x: Union[tuple, list]):
+
+def to_cartesian(x: Union[tuple, list, None]):
     if x is None:
         return None
     else:
         assert len(x) == 3
         return Cartesian.from_collection(x)
 
+
 class Cartesian(namedtuple('Cartesian', ['z', 'y', 'x'])):
     """Cartesian coordinate or offset."""
     __slots__ = ()
@@ -186,7 +188,7 @@ def inverse(self):
 
 
 @dataclass(frozen=True)
-class BoundingBox():
+class BoundingBox:
     start: Cartesian
     stop: Cartesian
     # def __post_init__(self, start, stop) -> BoundingBox:
@@ -361,7 +363,7 @@ def __floordiv__(self, other: Number | Cartesian | BoundingBox) -> BoundingBox:
             minpt = self.minpt // other.minpt
             maxpt = self.maxpt // other.maxpt
         elif isinstance(other, np.ndarray):
-            other  = Cartesian.from_collection(other)
+            other = Cartesian.from_collection(other)
             minpt = self.start // other
             maxpt = self.stop // other
         else:
@@ -387,7 +389,6 @@ def __iadd__(self, other: Cartesian | Number):
         stop = self.stop + other
         return BoundingBox(start, stop)
 
-
     def clone(self):
         return BoundingBox(self.start, self.stop)
 
@@ -468,9 +469,9 @@ def decompose(self, block_size: Cartesian,
 
         bboxes = BoundingBoxes()
 
-        for z in range(self.start.z, self.stop.z-block_size.z, block_size.z):
-            for y in range(self.start.y, self.stop.y-block_size.y, block_size.y):
-                for x in range(self.start.x, self.stop.x-block_size.x, block_size.x):
+        for z in range(self.start.z, self.stop.z-block_size.z+1, block_size.z):
+            for y in range(self.start.y, self.stop.y-block_size.y+1, block_size.y):
+                for x in range(self.start.x, self.stop.x-block_size.x+1, block_size.x):
                     bbox = BoundingBox.from_delta(Cartesian(z,y,x), block_size)
                     bboxes.append(bbox)
         return bboxes
@@ -489,7 +490,7 @@ def shape(self):
 
     @cached_property
     def left_neighbors(self):
-        sz = self.size3()
+        sz = self.shape
 
         minpt = deepcopy(self.minpt)
         minpt[0] -= sz[0]
@@ -695,20 +696,20 @@ def __len__(self):
 
 
 @dataclass(frozen=True)
-class PhysicalBoudingBox(BoundingBox):
+class PhysicalBoundingBox(BoundingBox):
     voxel_size: Cartesian
 
     @classmethod
     def from_bounding_box(cls, bbox: BoundingBox, 
-            voxel_size: Cartesian) -> PhysicalBoudingBox:
+            voxel_size: Cartesian) -> PhysicalBoundingBox:
         return cls(bbox.start, bbox.stop, 
             voxel_size)
 
     @cached_property
     def voxel_bounding_box(self) -> BoundingBox:
         return BoundingBox(self.start, self.stop)
 
-    def to_other_voxel_size(self, voxel_size2: Cartesian) -> PhysicalBoudingBox:
+    def to_other_voxel_size(self, voxel_size2: Cartesian) -> PhysicalBoundingBox:
         assert voxel_size2 != self.voxel_size
 
         if voxel_size2 >= self.voxel_size:
@@ -720,5 +721,4 @@ def to_other_voxel_size(self, voxel_size2: Cartesian) -> PhysicalBoudingBox:
             factors = self.voxel_size // voxel_size2
             start = self.start * factors
             stop = self.stop * factors
-        return PhysicalBoudingBox(start, stop, voxel_size2)
-
+        return PhysicalBoundingBox(start, stop, voxel_size2)

chunkflow/plugins/gaussian_filter.py

@@ -1,9 +1,12 @@
+import numpy as np
 from chunkflow.chunk import Chunk
 
 from scipy.ndimage import gaussian_filter
 
 
-def execute(chunk: Chunk, sigma: float=1.):
+def execute(chunk: Chunk, sigma: float=1., inplace=False):
+    if not inplace:
+        chunk = chunk.clone()
     for z in range(chunk.shape[-3]):
         if chunk.ndim == 4:
             for channel in range(chunk.shape[0]):

chunkflow/volume.py

@@ -11,7 +11,7 @@
 from cloudvolume import CloudVolume
 from chunkflow.lib.utils import str_to_dict
 from .lib.cartesian_coordinate import \
-    BoundingBox, Cartesian, BoundingBoxes, PhysicalBoudingBox
+    BoundingBox, Cartesian, BoundingBoxes, PhysicalBoundingBox
 from .chunk import Chunk
 
 
@@ -111,8 +111,8 @@ def block_size(self):
             self.vol.chunk_size[::-1])
 
     @cached_property 
-    def physical_bounding_box(self) -> PhysicalBoudingBox:
-        return PhysicalBoudingBox(
+    def physical_bounding_box(self) -> PhysicalBoundingBox:
+        return PhysicalBoundingBox(
             self.start, self.stop, self.voxel_size)
 
     @cached_property
@@ -296,7 +296,7 @@ def get_candidate_block_bounding_boxes_with_different_voxel_size(
             voxel_size_low
         )
         for bbox_low in pbbox_low.decompose(block_size_low):
-            pbbox_low = PhysicalBoudingBox(
+            pbbox_low = PhysicalBoundingBox(
                 bbox_low.start, bbox_low.stop, voxel_size_low)
             pbbox_high = pbbox_low.to_other_voxel_size(chunk.voxel_size)
             chunk_high = block_high.cutout(pbbox_high)

conftest.py

@@ -0,0 +1 @@
+pytest_plugins = ['chunkflow']