Skip to content

Commit

Permalink
fix unit test and command line errors after renaming
Browse files Browse the repository at this point in the history
  • Loading branch information
xiuliren committed Jul 13, 2022
1 parent d2884f2 commit 1286d49
Show file tree
Hide file tree
Showing 6 changed files with 43 additions and 20 deletions.
4 changes: 2 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ ChangeLog history
- added synapse annotation visualization.
- add read-json operator
- The code of neuroglancer operator is splitted to functions and becomes more modular. The code is more readable and maintainable.
- add zero-filling option for read-h5 operator. If there is no such file, we fill it with zeros.
- add zero-filling option for load-h5 operator. If there is no such file, we fill it with zeros.
- add skip-task operator. if a result file already exists, we skip the task
- add skip-all-zero operator. if a chunk is all zero, we skip that task.
- add new patch inference backend: prebuilt. [PR](https://github.com/seung-lab/chunkflow/pull/250)
Expand Down Expand Up @@ -87,7 +87,7 @@ ChangeLog history
- more options for generate-tasks operator
## Bug Fixes
- fix manifest by updating cloud storage to CloudFiles
- fix read-h5 operator when only cutout size is provided
- fix load-h5 operator when only cutout size is provided
- fix grid size bug of generating tasks
## Improved Documentation

Expand Down
1 change: 1 addition & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ After installation, You can simply type `chunkflow` and it will list all the ope

| Operator Name | Function |
| --------------- | -------- |
| adjust-bbox | adjust the corner offset of bounding box |
| aggregate-skeleton-fragments| Merge skeleton fragments from chunks |
| channel-voting | Vote across channels of semantic map |
| cleanup | remove empty files to clean up storage |
Expand Down
17 changes: 16 additions & 1 deletion chunkflow/flow/flow.py
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,21 @@ def generate_tasks(
yield task


@main.command('adjust-bbox')
@click.option('--corner-offset', '-c', type=click.INT, nargs=6, default=None,
help='adjust bounding box corner offset')
@operator
def adjust_bbox(tasks, corner_offset: tuple):
"""adjust the corner of bounding box."""
for task in tasks:
if task is not None:
bbox = task['bbox']
bbox.adjust_corner(corner_offset)
logging.info(f'after bounding box adjustment: {bbox.string}')
task['bbox'] = bbox
yield task


@main.command('skip-task')
@click.option('--prefix', '-p', required=True, type=str,
help='the pre part of result file path')
Expand Down Expand Up @@ -313,7 +328,7 @@ def skip_none(tasks: dict, input_name: str, touch: bool, prefix: str, suffix: st
help='Neuroglancer precomputed block compression algorithm.')
@click.option('--voxel-size', '-v', type=click.INT, nargs=3, default=(40, 4, 4),
help='voxel size or resolution of mip 0 image.')
@click.option('--oversave-info/--no-oversave-info', default=False,
@click.option('--overwrite-info/--no-overwrite-info', default=False,
help='normally we should avoid overwriting info file to avoid errors.')
@generator
def setup_env(volume_start, volume_stop, volume_size, layer_path,
Expand Down
1 change: 0 additions & 1 deletion chunkflow/flow/save_pngs.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ def __init__(self,

def __call__(self, chunk: Chunk):
assert isinstance(chunk, Chunk)
breakpoint()
if not np.issubdtype(chunk.dtype, self.dtype):
chunk = chunk.astype(self.dtype)

Expand Down
6 changes: 6 additions & 0 deletions chunkflow/lib/bounding_boxes.py
Original file line number Diff line number Diff line change
Expand Up @@ -287,6 +287,12 @@ def adjust(self, size: Union[Cartesian, int, tuple, list, Vec]):
self.maxpt += size[-3:]
return self

def adjust_corner(self, corner_offset: Union[tuple, list]):
if corner_offset is not None:
assert len(corner_offset) == 6
self.minpt += Cartesian.from_collection(corner_offset[:3])
self.maxpt += Cartesian.from_collection(corner_offset[-3:])

def union(self, bbox2):
"""Merge another bounding box
Expand Down
34 changes: 18 additions & 16 deletions tests/command_line.sh
Original file line number Diff line number Diff line change
Expand Up @@ -7,12 +7,12 @@ chunkflow generate-tasks -c 0 0 0 -s 0 0 0 -g 1 1 1
# echo "test nuclease..."
# chunkflow generate-tasks --roi-start 20789 21341 17019 --chunk-size 16 128 128 plugin -f cutout_dvid_label -i bbox -o chunk

chunkflow create-chunk --pattern zero skip-all-zero write-h5 -f /tmp/
chunkflow create-chunk --pattern zero skip-all-zero save-h5 -f /tmp/

echo "create a hdf5 file, then test the skip-task operator."
# the file already exist, so we'll skip that task
# if the skip is not successful, there is no chunk to write, we'll get an error.
chunkflow create-chunk write-h5 -f /tmp/
chunkflow create-chunk save-h5 -f /tmp/

echo "test skip tasks..."
chunkflow generate-tasks --roi-start 0 0 0 --chunk-size 64 64 64 skip-task --prefix /tmp/ --suffix .h5
Expand All @@ -28,27 +28,27 @@ echo "generate summary of log..."
chunkflow log-summary -l tests/data/log

echo "test write png files..."
chunkflow create-chunk write-pngs -o /tmp/pngs; rm -rf /tmp/pngs
chunkflow create-chunk save-pngs -o /tmp/pngs; rm -rf /tmp/pngs

echo "test normalize intensity..."
chunkflow create-chunk normalize-intensity

echo "connected component analysis..."
chunkflow create-chunk write-h5 --file-name="/tmp/img.h5" connected-components --threshold 128 write-h5 --file-name=/tmp/seg.h5
chunkflow create-chunk save-h5 --file-name="/tmp/img.h5" connected-components --threshold 128 save-h5 --file-name=/tmp/seg.h5
if test -f /tmp/img.h5 ; then echo "File found"; else exit 1; fi
chunkflow read-h5 --file-name=/tmp/img.h5
chunkflow load-h5 --file-name=/tmp/img.h5

echo "write and read nrrd file."
chunkflow create-chunk write-nrrd -f "/tmp/img.nrrd";
chunkflow read-nrrd -f "/tmp/img.nrrd";
chunkflow create-chunk save-nrrd -f "/tmp/img.nrrd";
chunkflow load-nrrd -f "/tmp/img.nrrd";
rm /tmp/img.nrrd;

echo "write and read tif file."
chunkflow create-chunk write-tif -f "/tmp/img.tif";
chunkflow read-tif -f "/tmp/img.tif";
chunkflow create-chunk save-tif -f "/tmp/img.tif";
chunkflow load-tif -f "/tmp/img.tif";
rm /tmp/img.tif;
chunkflow create-chunk --dtype uint16 --pattern sin write-tif -f /tmp/seg.tif
chunkflow read-tif -f /tmp/seg.tif
chunkflow create-chunk --dtype uint16 --pattern sin save-tif -f /tmp/seg.tif
chunkflow load-tif -f /tmp/seg.tif
rm /tmp/seg.tif

echo "create the info file of Neuroglancer Precomputed format."
Expand All @@ -61,16 +61,18 @@ echo "write image to precomputed volume."
# somehow, we have to separate creation of info and writing out precomputed operation!
chunkflow \
create-chunk --size 128 128 128 --dtype uint32 --pattern sin \
write-precomputed --volume-path file:///tmp/seg \
save-precomputed --volume-path file:///tmp/seg \
mesh --voxel-size 8 8 8 --output-format precomputed --output-path file:///tmp/seg
rm -rf /tmp/seg

echo "dry run of setup-environment in the cloud."
chunkflow --dry-run --log-level info\
chunkflow --dry-run --log-level info \
setup-env -l "gs://my/path" --volume-start 2002 25616 12304 \
--volume-stop 2068 26128 12816 --max-ram-size 14 --input-patch-size 20 128 128 \
--output-patch-size 16 96 96 --output-patch-overlap 6 32 32 --channel-num 3 \
--dtype float32 -m 0 --encoding raw --voxel-size 45 16 16 --max-mip 5
--volume-stop 2068 26128 12816 --max-ram-size 14 \
--input-patch-size 20 128 128 \
--output-patch-size 16 96 96 --output-patch-overlap 6 32 32 \
--channel-num 3 --dtype float32 -m 0 --encoding raw \
--voxel-size 45 16 16 --max-mip 5

echo "convolutional net inference."
chunkflow --log-level debug \
Expand Down

0 comments on commit 1286d49

Please sign in to comment.