master: Changes to exclude ktx/ folder

MouseLightProject · May 20, 2023 · 743a20a · 743a20a
1 parent 707cbb0
commit 743a20a
Show file tree

Hide file tree

Showing 25 changed files with 357 additions and 293 deletions.
diff --git a/batch_convert_leaf_mj2_files_to_tif.m b/batch_convert_leaf_mj2_files_to_tif.m
diff --git a/batch_convert_leaf_tif_files_to_mj2.m b/batch_convert_leaf_tif_files_to_mj2.m
diff --git a/..._preverify_tif_files_after_mj2_from_tif.m → batch_preverify_stack_files_after_freezing.m b/..._preverify_tif_files_after_mj2_from_tif.m → batch_preverify_stack_files_after_freezing.m
@@ -1,9 +1,9 @@
-function batch_preverify_tif_files_after_mj2_from_tif(mj2_folder_name, ...
-                                                      tif_folder_name, ...
-                                                      do_submit, ...
-                                                      do_try, ...
-                                                      submit_host_name, ...
-                                                      maximum_running_slot_count)
+function batch_preverify_stack_files_after_freezing(mj2_folder_name, ...
+                                                    tif_folder_name, ...
+                                                    do_submit, ...
+                                                    do_try, ...
+                                                    submit_host_name, ...
+                                                    maximum_running_slot_count)
     % Makes sure all the big target files are present in the the destination, and
     % are similar to their source files.
 

diff --git a/batch_verify_leaf_tif_files_after_thawing.m → batch_verify_tif_files_after_thawing.m b/batch_verify_leaf_tif_files_after_thawing.m → batch_verify_tif_files_after_thawing.m
@@ -1,16 +1,16 @@
-function batch_verify_leaf_tif_files_after_thawing(output_folder_name, ...
-                                                   input_folder_name, ...
-                                                   do_submit, ...
-                                                   do_try, ...
-                                                   submit_host_name, ...
-                                                   maximum_running_slot_count)
+function batch_verify_tif_files_after_thawing(output_folder_name, ...
+                                              input_folder_name, ...
+                                              do_submit, ...
+                                              do_try, ...
+                                              submit_host_name, ...
+                                              maximum_running_slot_count)
     % Makes sure all the leaf image files are present in the the destination, and
     % are similar to their source files.
 
     slots_per_job = 4 ;
-    bsub_options = '-P mouselight -W 59 -J verify-thawed-leaves' ;
+    bsub_options = '-P mouselight -W 59 -J verify-thawed-stack' ;
     find_and_batch(input_folder_name, ...
-                   @is_leaf_mj2_and_lacks_cert_for_output, ...
+                   @is_mj2_and_lacks_cert_for_output, ...
                    @verify_single_tif_file_after_thawing, ...
                    'do_submit', do_submit, ...
                    'do_try', do_try, ...

diff --git a/check_certs_after_verifying_thawing.m b/check_certs_after_verifying_thawing.m
@@ -1,10 +1,14 @@
 function check_certs_after_verifying_thawing(input_root_folder_path, output_root_folder_path)
     % Makes sure there's a .similar-tif-exists for each .mj2 file in
     % input_root_folder_path.
+
+    function result = check_single_cert_after_verifying_thawing_wrapper(root_folder_path, base_folder_relative_path, file_name, depth)  %#ok<INUSD> 
+        result = check_single_cert_after_verifying_thawing(root_folder_path, base_folder_relative_path, file_name, output_root_folder_path) ;
+    end
+
     find_and_verify(...
         input_root_folder_path, ...
-        @is_a_mj2_in_a_folder_with_no_subfolders, ...
-        @check_single_cert_after_verifying_thawing, ...
-        input_root_folder_path, ...
-        output_root_folder_path) ;
+        @does_file_name_end_in_dot_mj2, ...
+        @check_single_cert_after_verifying_thawing_wrapper, ...
+        @(varargin)(true)) ;
 end
diff --git a/check_single_cert_after_verifying_thawing.m b/check_single_cert_after_verifying_thawing.m
@@ -1,7 +1,7 @@
-function result = check_single_cert_after_verifying_thawing(input_file_path, input_root_folder_path, output_root_folder_path)
-    input_relative_file_path = relpath(input_file_path, input_root_folder_path) ;
-    output_relative_file_path = replace_extension(input_relative_file_path, '.tif') ;
-    output_file_path = fullfile(output_root_folder_path, output_relative_file_path) ;
+function result = check_single_cert_after_verifying_thawing(...
+        input_root_folder_path, input_folder_relative_path, input_file_name, output_root_folder_path)  %#ok<INUSD> 
+    output_file_name = replace_extension(input_file_name, '.tif') ;
+    output_file_path = fullfile(output_root_folder_path, input_folder_relative_path, output_file_name) ;
     cert_file_path = horzcat(output_file_path, '.is-similar-to-mj2') ;    
     result = logical( exist(cert_file_path, 'file') ) ;
 end
diff --git a/copy_non_stack_files_during_freezing.m b/copy_non_stack_files_during_freezing.m
@@ -0,0 +1,30 @@
+function copy_non_stack_files_during_freezing(output_folder_name, ...
+                                              input_folder_name)
+    % Find all non-.tif files, copy them over to the mj2 side
+
+    do_submit = false ;
+    do_try = true ;
+    maximum_running_slot_count = 400 ;  % doesn't matter
+    slots_per_job = 1 ;  % doesn't matter
+    bsub_options = '-P mouselight -W 59 -J copy' ;  % doesn't matter
+    submit_host_name = '' ;
+
+    function result = predicate_function(tif_side_file_path)
+        result = is_a_non_tif_and_target_missing(tif_side_file_path, input_folder_name, output_folder_name) ;
+    end
+
+    function batch_function(tif_side_file_path)
+        copy_file_for_find_and_batch(tif_side_file_path, input_folder_name, output_folder_name) ;
+    end
+
+    find_and_batch(input_folder_name, ...
+                   @predicate_function, ...
+                   @batch_function, ...
+                   'do_submit', do_submit, ...
+                   'do_try', do_try, ...
+                   'submit_host_name', submit_host_name, ...
+                   'maximum_running_slot_count', maximum_running_slot_count, ...
+                   'slots_per_job', slots_per_job, ...
+                   'bsub_options', bsub_options, ...
+                   'folder_predicate_function', @is_not_the_ktx_folder) ;
+end
diff --git a/copy_non_mj2_files_to_tif_side.m → copy_non_stack_files_during_thawing.m b/copy_non_mj2_files_to_tif_side.m → copy_non_stack_files_during_thawing.m
@@ -1,5 +1,5 @@
-function copy_non_mj2_files_to_tif_side(output_folder_name, ...
-                                        input_folder_name)
+function copy_non_stack_files_during_thawing(output_folder_name, ...
+                                             input_folder_name)
     % Find all non-.mj2 files, copy them over to the tif side
 
     do_submit = false ;
@@ -8,15 +8,22 @@ function copy_non_mj2_files_to_tif_side(output_folder_name, ...
     slots_per_job = 1 ;  % doesn't matter
     bsub_options = '-P mouselight -W 59 -J copy' ;  % doesn't matter
     submit_host_name = '' ;
+
+    function result = predicate_function(mj2_side_file_path)
+        result = is_a_non_mj2_and_target_missing(mj2_side_file_path, input_folder_name, output_folder_name) ;
+    end
+
+    function batch_function(mj2_side_file_path)
+        copy_file_for_find_and_batch(mj2_side_file_path, input_folder_name, output_folder_name) ;
+    end    
+
     find_and_batch(input_folder_name, ...
-                   @is_a_non_mj2_and_target_missing, ...
-                   @copy_file_for_find_and_batch, ...
+                   @predicate_function, ...
+                   @batch_function, ...
                    'do_submit', do_submit, ...
                    'do_try', do_try, ...
                    'submit_host_name', submit_host_name, ...
                    'maximum_running_slot_count', maximum_running_slot_count, ...
                    'slots_per_job', slots_per_job, ...
-                   'bsub_options', bsub_options, ...
-                   'predicate_extra_args', {input_folder_name, output_folder_name}, ...
-                   'batch_function_extra_args', {input_folder_name, output_folder_name}) ;
+                   'bsub_options', bsub_options) ;
 end
diff --git a/copy_non_tif_files_to_mj2_side.m b/copy_non_tif_files_to_mj2_side.m
diff --git a/does_is_similar_to_mj2_check_file_exist.m b/does_is_similar_to_mj2_check_file_exist.m
@@ -0,0 +1,7 @@
+function result = does_is_similar_to_mj2_check_file_exist(tif_file_path, mj2_root_path, tif_root_path)
+    mj2_relative_path = relpath(tif_file_path, mj2_root_path) ;
+    tif_relative_path = replace_extension(mj2_relative_path, '.tif') ;
+    tif_file_path = fullfile(tif_root_path, tif_relative_path) ;
+    check_file_path = horzcat(tif_file_path, '.is-similar-to-mj2') ;
+    result = logical(exist(check_file_path, 'file')) ;
+end
diff --git a/does_need_preverification_after_freezing.m b/does_need_preverification_after_freezing.m
@@ -1,4 +1,4 @@
 function result = does_need_preverification_after_freezing(tif_file_path, tif_root_path, mj2_root_path)
-    result = is_a_tif_and_target_present(tif_file_path, tif_root_path, mj2_root_path) && ...
+    result = does_file_name_end_in_dot_tif(tif_file_path) && ...
              ~does_is_similar_to_tif_check_file_exist(tif_file_path, tif_root_path, mj2_root_path) ;
 end
diff --git a/find_and_count.m b/find_and_count.m
@@ -1,37 +1,70 @@
-function result = find_and_count(base_folder_path, is_file_a_keeper_predicate, varargin)
-    result = ...
-        find_and_list_helper(base_folder_path, ...
-                             is_file_a_keeper_predicate, ...
-                             0, ...
-                             varargin{:}) ;
-end
+function result = find_and_count(root_folder_path, is_file_a_keeper_predicate, folder_filter_function)
+    seed = 0 ;  % for counting, start with a count of zero
+
+    function result = filter(root_folder_path, base_folder_relative_path, file_name, depth, is_this_file_a_folder)  %#ok<INUSD>
+        result = feval(is_file_a_keeper_predicate, root_folder_path, base_folder_relative_path, file_name) ;
+    end
 
+    result = file_accumulate_map_filter(root_folder_path, seed, @accumulator, @mapper, @filter, folder_filter_function) ;
+end
 
+function result = accumulator(result_so_far, file_value)
+    % For counting, just add the file_value to the ongoing count
+    result = result_so_far + file_value ;
+end
 
-function result = ...
-        find_and_list_helper(base_folder_path, ...
-                             is_file_a_keeper_predicate, ...
-                             incoming_count, ...
-                             varargin)
-    count_so_far = incoming_count ;
-    [file_names, is_file_a_folder] = simple_dir(base_folder_path) ;
-    file_count = length(file_names) ;
-    for i = 1 : file_count ,
-        file_name = file_names{i} ;
-        is_this_file_a_folder = is_file_a_folder(i) ;
-        file_path = fullfile(base_folder_path, file_name) ;
-        if is_this_file_a_folder ,
-            % if a folder, recurse
-            count_so_far = ...
-                find_and_list_helper(file_path, ...
-                                     is_file_a_keeper_predicate, ...
-                                     count_so_far, ...
-                                     varargin{:}) ;
-        else
-            if feval(is_file_a_keeper_predicate, file_path, varargin{:}) ,
-                count_so_far = count_so_far + 1 ;
-            end
-        end
-    end    
-    result = count_so_far ;
+function result = mapper(root_folder_path, base_folder_relative_path, file_name, depth)  %#ok<INUSD> 
+    % For counting, just want to return one for each file that gets past the
+    % filter
+    result = 1 ;
 end
+
+
+% function result = find_and_count(base_folder_path, is_file_a_keeper_predicate, varargin)
+%     [folder_predicate_function] = ...
+%         parse_keyword_args(...
+%         varargin, ...
+%         'folder_predicate_function', @(folder_path, depth)(true)) ;
+%     
+%     result = ...
+%         find_and_count_helper(base_folder_path, ...
+%                               is_file_a_keeper_predicate, ...
+%                               folder_predicate_function, ...
+%                               0, ...
+%                               0) ;
+% end
+% 
+% 
+% 
+% function result = ...
+%         find_and_count_helper(base_folder_path, ...
+%                               is_file_a_keeper_predicate, ...
+%                               folder_predicate_function, ...
+%                               depth, ...
+%                               incoming_count)
+%     count_so_far = incoming_count ;
+%     [file_names, is_file_a_folder] = simple_dir(base_folder_path) ;
+%     file_count = length(file_names) ;
+%     for i = 1 : file_count ,
+%         file_name = file_names{i} ;
+%         is_this_file_a_folder = is_file_a_folder(i) ;
+%         file_path = fullfile(base_folder_path, file_name) ;
+%         if is_this_file_a_folder ,
+%             % if a folder satisfying predicate function, recurse
+%             if feval(folder_predicate_function, file_path, depth) ,
+%                 count_so_far = ...
+%                     find_and_count_helper(...
+%                         file_path, ...
+%                         is_file_a_keeper_predicate, ...
+%                         folder_predicate_function, ...
+%                         depth+1, ...
+%                         count_so_far) ;
+%             end
+%         else
+%             if feval(is_file_a_keeper_predicate, file_path) ,
+%                 count_so_far = count_so_far + 1 ;
+%             end
+%         end
+%     end    
+%     result = count_so_far ;
+% end