diff --git a/scripts/atomic_chunk_ws.sh b/scripts/atomic_chunk_ws.sh index db82698..ddb10f8 100755 --- a/scripts/atomic_chunk_ws.sh +++ b/scripts/atomic_chunk_ws.sh @@ -8,7 +8,7 @@ try acquire_cpu_slot try mkdir remap try mkdir -p ${output_path}/{seg,dend} try taskset -c $cpuid python3 $SCRIPT_PATH/cut_chunk_ws.py $1 -try taskset -c $cpuid $BIN_PATH/ws param.txt aff.raw $WS_HIGH_THRESHOLD $WS_LOW_THRESHOLD $WS_SIZE_THRESHOLD $output_chunk +try taskset -c $cpuid $BIN_PATH/ws param.txt aff.raw $WS_HIGH_THRESHOLD $WS_LOW_THRESHOLD $WS_SIZE_THRESHOLD $WS_DUST_THRESHOLD $output_chunk try touch ongoing_"${output_chunk}".data try $COMPRESS_CMD seg_"${output_chunk}".data try mv seg_"${output_chunk}".data."${COMPRESSED_EXT}" ${output_path}/seg/ diff --git a/scripts/composite_chunk_ws.sh b/scripts/composite_chunk_ws.sh index 61b8a9a..cb66393 100755 --- a/scripts/composite_chunk_ws.sh +++ b/scripts/composite_chunk_ws.sh @@ -13,7 +13,7 @@ try download_children $1 $FILE_PATH/dend try python3 $SCRIPT_PATH/merge_chunks_ws.py $1 #try $BIN_PATH/ws2 param.txt $output_chunk >& debug_"${output_chunk}".log -try $BIN_PATH/ws2 param.txt $WS_HIGH_THRESHOLD $WS_LOW_THRESHOLD $WS_SIZE_THRESHOLD $output_chunk +try $BIN_PATH/ws2 param.txt $WS_HIGH_THRESHOLD $WS_LOW_THRESHOLD $WS_SIZE_THRESHOLD $WS_DUST_THRESHOLD $output_chunk try mv done_{pre,post}_"${output_chunk}".data ${output_path}/remap diff --git a/scripts/set_env.py b/scripts/set_env.py index ea70c04..54934b9 100644 --- a/scripts/set_env.py +++ b/scripts/set_env.py @@ -21,7 +21,7 @@ def default_io_cmd(path): return "cloudfiles cp -r" -env = ["SCRATCH_PATH", "CHUNKMAP_INPUT", "CHUNKMAP_OUTPUT", "AFF_PATH", "AFF_MIP", "SEM_PATH", "SEM_MIP", "WS_PATH", "SEG_PATH", "WS_HIGH_THRESHOLD", "WS_LOW_THRESHOLD", "WS_SIZE_THRESHOLD", "AGG_THRESHOLD", "GT_PATH", "CLEFT_PATH", "MYELIN_THRESHOLD", "ADJUSTED_AFF_PATH", "CHUNKED_AGG_OUTPUT", "CHUNKED_SEG_PATH", "REDIS_SERVER", "REDIS_DB", "STATSD_HOST", "STATSD_PORT", "STATSD_PREFIX", "PARANOID", "BOTO_CONFIG", "UPLOAD_CMD", "DOWNLOAD_CMD", "IO_SCRATCH_PATH"] +env = ["SCRATCH_PATH", "CHUNKMAP_INPUT", "CHUNKMAP_OUTPUT", "AFF_PATH", "AFF_MIP", "SEM_PATH", "SEM_MIP", "WS_PATH", "SEG_PATH", "WS_HIGH_THRESHOLD", "WS_LOW_THRESHOLD", "WS_SIZE_THRESHOLD", "WS_DUST_THRESHOLD", "AGG_THRESHOLD", "GT_PATH", "CLEFT_PATH", "MYELIN_THRESHOLD", "ADJUSTED_AFF_PATH", "CHUNKED_AGG_OUTPUT", "CHUNKED_SEG_PATH", "REDIS_SERVER", "REDIS_DB", "STATSD_HOST", "STATSD_PORT", "STATSD_PREFIX", "PARANOID", "BOTO_CONFIG", "UPLOAD_CMD", "DOWNLOAD_CMD", "IO_SCRATCH_PATH"] with open(sys.argv[1]) as f: data = json.load(f) diff --git a/src/ws/atomic_chunk.cpp b/src/ws/atomic_chunk.cpp index 737c955..415d246 100644 --- a/src/ws/atomic_chunk.cpp +++ b/src/ws/atomic_chunk.cpp @@ -62,12 +62,14 @@ int main(int argc, char* argv[]) std::string ht(argv[3]); std::string lt(argv[4]); std::string st(argv[5]); - std::cout << "thresholds: "<< ht << " " << lt << " " << st << std::endl; + std::string dt(argv[6]); + std::cout << "thresholds: "<< ht << " " << lt << " " << st << " " << dt << std::endl; - const char * tag = argv[6]; + const char * tag = argv[7]; auto high_threshold = read_float(ht); auto low_threshold = read_float(lt); auto size_threshold = read_int(st); + auto dust_threshold = read_int(dt); param_file >> xdim >> ydim >> zdim; std::cout << xdim << " " << ydim << " " << zdim << std::endl; @@ -120,7 +122,7 @@ int main(int argc, char* argv[]) std::cout << "finished region graph in " << elapsed_secs << " seconds" << std::endl; begin = clock(); - merge_segments(seg, rg, counts, std::make_pair(size_threshold, low_threshold), size_threshold); + merge_segments(seg, rg, counts, std::make_pair(size_threshold, low_threshold), dust_threshold); end = clock(); elapsed_secs = double(end - begin) / CLOCKS_PER_SEC; diff --git a/src/ws/merge_chunks.cpp b/src/ws/merge_chunks.cpp index 37f1420..78023cf 100644 --- a/src/ws/merge_chunks.cpp +++ b/src/ws/merge_chunks.cpp @@ -53,7 +53,7 @@ region_graph load_dend(size_t data_size) template std::tuple>, size_t, size_t> -process_chunk_borders(size_t face_size, std::vector > & size_pairs, region_graph & rg, auto high_threshold, auto low_threshold, auto size_threshold, const std::string & tag, size_t remap_size, size_t ac_offset) +process_chunk_borders(size_t face_size, std::vector > & size_pairs, region_graph & rg, auto high_threshold, auto low_threshold, auto size_threshold, auto dust_threshold, const std::string & tag, size_t remap_size, size_t ac_offset) { std::vector sizes; std::vector segids; @@ -379,11 +379,11 @@ process_chunk_borders(size_t face_size, std::vector > & si for (size_t v = 0; v != sizes.size(); v++) { size_t size = sizes[v]; const ID s = sets.find_set(v); - if (sizes[s] >= size_threshold) { + if (sizes[s] >= dust_threshold) { remaps[v] = s; } - if ( (size & (~traits::on_border)) && size >= size_threshold ) { + if ( (size & (~traits::on_border)) && size >= dust_threshold ) { if (s != v) { std::cout << "s("<(ht); auto low_threshold = read_float(lt); auto size_threshold = read_int(st); + auto dust_threshold = read_int(dt); param_file >> xdim >> ydim >> zdim; std::cout << xdim << " " << ydim << " " << zdim << std::endl; std::array flags({true,true,true,true,true,true}); @@ -727,7 +729,7 @@ int main(int argc, char* argv[]) size_t c = 0; size_t d = 0; - std::tie(remaps, c, d) = process_chunk_borders(face_size, sizes, dend, high_threshold, low_threshold, size_threshold, tag, remap_size, ac_offset); + std::tie(remaps, c, d) = process_chunk_borders(face_size, sizes, dend, high_threshold, low_threshold, size_threshold, dust_threshold, tag, remap_size, ac_offset); update_border_supervoxels(remaps, flags, std::array({ydim*zdim, xdim*zdim, xdim*ydim, ydim*zdim, xdim*zdim, xdim*ydim}), tag); //auto m = write_remap(remaps, tag); std::vector meta({xdim,ydim,zdim,c,d,0});