Resurrect the dust parameter

seung-lab · Sep 19, 2024 · 8e1d741 · 8e1d741
1 parent 278ff58
commit 8e1d741
Show file tree

Hide file tree

Showing 5 changed files with 16 additions and 12 deletions.
diff --git a/scripts/atomic_chunk_ws.sh b/scripts/atomic_chunk_ws.sh
@@ -8,7 +8,7 @@ try acquire_cpu_slot
 try mkdir remap
 try mkdir -p ${output_path}/{seg,dend}
 try taskset -c $cpuid python3 $SCRIPT_PATH/cut_chunk_ws.py $1
-try taskset -c $cpuid $BIN_PATH/ws param.txt aff.raw $WS_HIGH_THRESHOLD $WS_LOW_THRESHOLD $WS_SIZE_THRESHOLD $output_chunk
+try taskset -c $cpuid $BIN_PATH/ws param.txt aff.raw $WS_HIGH_THRESHOLD $WS_LOW_THRESHOLD $WS_SIZE_THRESHOLD $WS_DUST_THRESHOLD $output_chunk
 try touch ongoing_"${output_chunk}".data
 try $COMPRESS_CMD seg_"${output_chunk}".data
 try mv seg_"${output_chunk}".data."${COMPRESSED_EXT}" ${output_path}/seg/

diff --git a/scripts/composite_chunk_ws.sh b/scripts/composite_chunk_ws.sh
@@ -13,7 +13,7 @@ try download_children $1 $FILE_PATH/dend
 
 try python3 $SCRIPT_PATH/merge_chunks_ws.py $1
 #try $BIN_PATH/ws2 param.txt $output_chunk >& debug_"${output_chunk}".log
-try $BIN_PATH/ws2 param.txt $WS_HIGH_THRESHOLD $WS_LOW_THRESHOLD $WS_SIZE_THRESHOLD $output_chunk
+try $BIN_PATH/ws2 param.txt $WS_HIGH_THRESHOLD $WS_LOW_THRESHOLD $WS_SIZE_THRESHOLD $WS_DUST_THRESHOLD $output_chunk
 
 try mv done_{pre,post}_"${output_chunk}".data ${output_path}/remap
 

diff --git a/scripts/set_env.py b/scripts/set_env.py
@@ -21,7 +21,7 @@ def default_io_cmd(path):
         return "cloudfiles cp -r"
 
 
-env = ["SCRATCH_PATH", "CHUNKMAP_INPUT", "CHUNKMAP_OUTPUT", "AFF_PATH", "AFF_MIP", "SEM_PATH", "SEM_MIP", "WS_PATH", "SEG_PATH", "WS_HIGH_THRESHOLD", "WS_LOW_THRESHOLD", "WS_SIZE_THRESHOLD", "AGG_THRESHOLD", "GT_PATH", "CLEFT_PATH", "MYELIN_THRESHOLD", "ADJUSTED_AFF_PATH", "CHUNKED_AGG_OUTPUT", "CHUNKED_SEG_PATH", "REDIS_SERVER", "REDIS_DB", "STATSD_HOST", "STATSD_PORT", "STATSD_PREFIX", "PARANOID", "BOTO_CONFIG", "UPLOAD_CMD", "DOWNLOAD_CMD", "IO_SCRATCH_PATH"]
+env = ["SCRATCH_PATH", "CHUNKMAP_INPUT", "CHUNKMAP_OUTPUT", "AFF_PATH", "AFF_MIP", "SEM_PATH", "SEM_MIP", "WS_PATH", "SEG_PATH", "WS_HIGH_THRESHOLD", "WS_LOW_THRESHOLD", "WS_SIZE_THRESHOLD", "WS_DUST_THRESHOLD", "AGG_THRESHOLD", "GT_PATH", "CLEFT_PATH", "MYELIN_THRESHOLD", "ADJUSTED_AFF_PATH", "CHUNKED_AGG_OUTPUT", "CHUNKED_SEG_PATH", "REDIS_SERVER", "REDIS_DB", "STATSD_HOST", "STATSD_PORT", "STATSD_PREFIX", "PARANOID", "BOTO_CONFIG", "UPLOAD_CMD", "DOWNLOAD_CMD", "IO_SCRATCH_PATH"]
 
 with open(sys.argv[1]) as f:
     data = json.load(f)

diff --git a/src/ws/atomic_chunk.cpp b/src/ws/atomic_chunk.cpp
@@ -62,12 +62,14 @@ int main(int argc, char* argv[])
     std::string ht(argv[3]);
     std::string lt(argv[4]);
     std::string st(argv[5]);
-    std::cout << "thresholds: "<< ht << " " << lt << " " << st << std::endl;
+    std::string dt(argv[6]);
+    std::cout << "thresholds: "<< ht << " " << lt << " " << st  << " " << dt << std::endl;
 
-    const char * tag = argv[6];
+    const char * tag = argv[7];
     auto high_threshold = read_float<aff_t>(ht);
     auto low_threshold = read_float<aff_t>(lt);
     auto size_threshold = read_int(st);
+    auto dust_threshold = read_int(dt);
     param_file >> xdim >> ydim >> zdim;
     std::cout << xdim << " " << ydim << " " << zdim << std::endl;
 
@@ -120,7 +122,7 @@ int main(int argc, char* argv[])
     std::cout << "finished region graph in " << elapsed_secs << " seconds" << std::endl;
 
     begin = clock();
-    merge_segments(seg, rg, counts, std::make_pair(size_threshold, low_threshold), size_threshold);
+    merge_segments(seg, rg, counts, std::make_pair(size_threshold, low_threshold), dust_threshold);
     end = clock();
     elapsed_secs = double(end - begin) / CLOCKS_PER_SEC;
 

diff --git a/src/ws/merge_chunks.cpp b/src/ws/merge_chunks.cpp
@@ -53,7 +53,7 @@ region_graph<ID,F> load_dend(size_t data_size)
 
 template<typename ID, typename F>
 std::tuple<std::vector<std::pair<ID, ID>>, size_t, size_t>
-process_chunk_borders(size_t face_size, std::vector<std::pair<ID, size_t> > & size_pairs, region_graph<ID, F> & rg, auto high_threshold, auto low_threshold, auto size_threshold, const std::string & tag, size_t remap_size, size_t ac_offset)
+process_chunk_borders(size_t face_size, std::vector<std::pair<ID, size_t> > & size_pairs, region_graph<ID, F> & rg, auto high_threshold, auto low_threshold, auto size_threshold, auto dust_threshold, const std::string & tag, size_t remap_size, size_t ac_offset)
 {
     std::vector<size_t> sizes;
     std::vector<ID> segids;
@@ -379,11 +379,11 @@ process_chunk_borders(size_t face_size, std::vector<std::pair<ID, size_t> > & si
     for (size_t v = 0; v != sizes.size(); v++) {
         size_t size = sizes[v];
         const ID s = sets.find_set(v);
-        if (sizes[s] >= size_threshold) {
+        if (sizes[s] >= dust_threshold) {
             remaps[v] = s;
         }
 
-        if ( (size & (~traits::on_border)) && size >= size_threshold  ) {
+        if ( (size & (~traits::on_border)) && size >= dust_threshold  ) {
             if (s != v) {
                 std::cout << "s("<<s<<") != v("<<v<<")" << std::endl;
             }
@@ -672,11 +672,13 @@ int main(int argc, char* argv[])
     std::string ht(argv[2]);
     std::string lt(argv[3]);
     std::string st(argv[4]);
-    std::cout << "thresholds: "<< ht << " " << lt << " " << st << std::endl;
-    const char * tag = argv[5];
+    std::string dt(argv[5]);
+    std::cout << "thresholds: "<< ht << " " << lt << " " << st << " " << dt << std::endl;
+    const char * tag = argv[6];
     auto high_threshold = read_float<aff_t>(ht);
     auto low_threshold = read_float<aff_t>(lt);
     auto size_threshold = read_int(st);
+    auto dust_threshold = read_int(dt);
     param_file >> xdim >> ydim >> zdim;
     std::cout << xdim << " " << ydim << " " << zdim << std::endl;
     std::array<bool,6> flags({true,true,true,true,true,true});
@@ -727,7 +729,7 @@ int main(int argc, char* argv[])
     size_t c = 0;
     size_t d = 0;
 
-    std::tie(remaps, c, d) = process_chunk_borders<seg_t, aff_t>(face_size, sizes, dend, high_threshold, low_threshold, size_threshold, tag, remap_size, ac_offset);
+    std::tie(remaps, c, d) = process_chunk_borders<seg_t, aff_t>(face_size, sizes, dend, high_threshold, low_threshold, size_threshold, dust_threshold, tag, remap_size, ac_offset);
     update_border_supervoxels(remaps, flags, std::array<size_t, 6>({ydim*zdim, xdim*zdim, xdim*ydim, ydim*zdim, xdim*zdim, xdim*ydim}), tag);
     //auto m = write_remap(remaps, tag);
     std::vector<size_t> meta({xdim,ydim,zdim,c,d,0});