master: Replaced with code from octree repo, b/c more modern

MouseLightProject · May 18, 2023 · d3b4017 · d3b4017
1 parent f2cd8a3
commit d3b4017
Show file tree

Hide file tree

Showing 125 changed files with 783 additions and 1,166 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "mouselight_toolbox"]
+	path = mouselight_toolbox
+	url = https://github.com/MouseLightProject/mouselight_toolbox
diff --git a/are_files_same_size_and_md5sum.m b/are_files_same_size_and_md5sum.m
@@ -0,0 +1,9 @@
+function result = are_files_same_size_and_md5sum(file_name_1, file_name_2)
+    file_size_1 = get_file_size(file_name_1) ;
+    file_size_2 = get_file_size(file_name_2) ;
+    if file_size_1 ~= file_size_2 ,
+        result = false ;
+        return
+    end
+    result = are_md5s_the_same(file_name_1, file_name_2) ;
+end
diff --git a/batch_convert_all_mj2_files_to_tif.m b/batch_convert_all_mj2_files_to_tif.m
@@ -0,0 +1,22 @@
+function batch_convert_all_mj2_files_to_tif(output_folder_name, ...
+                                            input_folder_name, ...
+                                            do_submit, ...
+                                            do_try, ...
+                                            maximum_running_slot_count, ...
+                                            submit_host_name)
+    % Find all .mj2 files in folders with no subfolders, convert them to .tif
+
+    slots_per_job = 1 ;
+    bsub_options = '-P mouselight -W 59 -J thaw' ;
+    find_and_batch(input_folder_name, ...
+                   @is_a_mj2_and_target_missing, ...
+                   @tif_from_mj2_single_for_find_and_batch, ...
+                   'do_submit', do_submit, ...
+                   'do_try', do_try, ...
+                   'submit_host_name', submit_host_name, ...
+                   'maximum_running_slot_count', maximum_running_slot_count, ...
+                   'slots_per_job', slots_per_job, ...
+                   'bsub_options', bsub_options, ...
+                   'predicate_extra_args', {input_folder_name, output_folder_name}, ...
+                   'batch_function_extra_args', {input_folder_name, output_folder_name}) ;
+end
diff --git a/batch_convert_all_tif_files_to_mj2.m b/batch_convert_all_tif_files_to_mj2.m
@@ -0,0 +1,23 @@
+function batch_convert_all_tif_files_to_mj2(mj2_folder_name, ...
+                                            tif_folder_name, ...
+                                            compression_ratio, ...
+                                            do_submit, ...
+                                            do_try, ...
+                                            maximum_running_slot_count, ...
+                                            submit_host_name)
+    % Find all .tif files in folder tree, convert them to .mj2
+
+    slots_per_job = 1 ;
+    bsub_options = '-P mouselight -W 59 -J freeze' ;
+    find_and_batch(tif_folder_name, ...
+                   @does_file_name_end_in_dot_tif_and_mj2_is_missing, ...
+                   @mj2_from_tif_single_for_find_and_batch, ...
+                   'do_submit', do_submit, ...
+                   'do_try', do_try, ...
+                   'submit_host_name', submit_host_name, ...
+                   'maximum_running_slot_count', maximum_running_slot_count, ...
+                   'slots_per_job', slots_per_job, ...
+                   'bsub_options', bsub_options, ...
+                   'predicate_extra_args', {tif_folder_name, mj2_folder_name}, ...
+                   'batch_function_extra_args', {tif_folder_name, mj2_folder_name, compression_ratio}) ;
+end
diff --git a/batch_convert_leaf_mj2_files_to_tif.m b/batch_convert_leaf_mj2_files_to_tif.m
@@ -0,0 +1,22 @@
+function batch_convert_leaf_mj2_files_to_tif(output_folder_name, ...
+                                             input_folder_name, ...
+                                             do_submit, ...
+                                             do_try, ...
+                                             maximum_running_slot_count, ...
+                                             submit_host_name)
+    % Find all .mj2 files in folders with no subfolders, convert them to .tif
+
+    slots_per_job = 1 ;
+    bsub_options = '-P mouselight -W 59 -J thaw' ;
+    find_and_batch(input_folder_name, ...
+                   @is_a_mj2_in_a_folder_with_no_subfolders_and_target_missing, ...
+                   @tif_from_mj2_single_for_find_and_batch, ...
+                   'do_submit', do_submit, ...
+                   'do_try', do_try, ...
+                   'submit_host_name', submit_host_name, ...
+                   'maximum_running_slot_count', maximum_running_slot_count, ...
+                   'slots_per_job', slots_per_job, ...
+                   'bsub_options', bsub_options, ...
+                   'predicate_extra_args', {input_folder_name, output_folder_name}, ...
+                   'batch_function_extra_args', {input_folder_name, output_folder_name}) ;
+end
diff --git a/batch_convert_leaf_tif_files_to_mj2.m b/batch_convert_leaf_tif_files_to_mj2.m
@@ -0,0 +1,23 @@
+function batch_convert_leaf_tif_files_to_mj2(mj2_folder_name, ...
+                                             tif_folder_name, ...
+                                             compression_ratio, ...
+                                             do_submit, ...
+                                             do_try, ...
+                                             maximum_running_slot_count, ...
+                                             submit_host_name)
+    % Find all .tif files in folders with no subfolders, convert them to .mj2
+
+    slots_per_job = 1 ;
+    bsub_options = '-P mouselight -W 59 -J freeze' ;
+    find_and_batch(tif_folder_name, ...
+                   @is_a_tif_in_a_folder_with_no_subfolders_and_target_missing, ...
+                   @mj2_from_tif_single_for_find_and_batch, ...
+                   'do_submit', do_submit, ...
+                   'do_try', do_try, ...
+                   'submit_host_name', submit_host_name, ...
+                   'maximum_running_slot_count', maximum_running_slot_count, ...
+                   'slots_per_job', slots_per_job, ...
+                   'bsub_options', bsub_options, ...
+                   'predicate_extra_args', {tif_folder_name, mj2_folder_name}, ...
+                   'batch_function_extra_args', {tif_folder_name, mj2_folder_name, compression_ratio}) ;
+end
diff --git a/batch_preverify_tif_files_after_mj2_from_tif.m b/batch_preverify_tif_files_after_mj2_from_tif.m
@@ -1,4 +1,9 @@
-function batch_preverify_tif_files_after_mj2_from_tif(mj2_folder_name, tif_folder_name, do_submit, maximum_running_slot_count)
+function batch_preverify_tif_files_after_mj2_from_tif(mj2_folder_name, ...
+                                                      tif_folder_name, ...
+                                                      do_submit, ...
+                                                      do_try, ...
+                                                      submit_host_name, ...
+                                                      maximum_running_slot_count)
     % Makes sure all the big target files are present in the the destination, and
     % are similar to their source files.
 
@@ -7,10 +12,12 @@ function batch_preverify_tif_files_after_mj2_from_tif(mj2_folder_name, tif_folde
     find_and_batch(tif_folder_name, ...
                    @does_need_preverification_after_freezing, ...
                    @preverify_single_tif_file_after_mj2_from_tif, ...
-                   do_submit, ...
-                   maximum_running_slot_count, ...
-                   slots_per_job, ...
-                   bsub_options, ...
-                   tif_folder_name, ...
-                   mj2_folder_name) ;
+                   'do_submit', do_submit, ...
+                   'do_try', do_try, ...
+                   'submit_host_name', submit_host_name, ...
+                   'maximum_running_slot_count', maximum_running_slot_count, ...
+                   'slots_per_job', slots_per_job, ...
+                   'bsub_options', bsub_options, ...
+                   'predicate_extra_args', {tif_folder_name, mj2_folder_name}, ...
+                   'batch_function_extra_args', {tif_folder_name, mj2_folder_name}) ;
 end
diff --git a/batch_preverify_tif_files_for_2019_03_26_frozen_raw_tiles.m b/batch_preverify_tif_files_for_2019_03_26_frozen_raw_tiles.m
diff --git a/batch_verify_leaf_tif_files_after_thawing.m b/batch_verify_leaf_tif_files_after_thawing.m
@@ -0,0 +1,23 @@
+function batch_verify_leaf_tif_files_after_thawing(output_folder_name, ...
+                                                   input_folder_name, ...
+                                                   do_submit, ...
+                                                   do_try, ...
+                                                   submit_host_name, ...
+                                                   maximum_running_slot_count)
+    % Makes sure all the leaf image files are present in the the destination, and
+    % are similar to their source files.
+
+    slots_per_job = 4 ;
+    bsub_options = '-P mouselight -W 59 -J verify-thawed-leaves' ;
+    find_and_batch(input_folder_name, ...
+                   @is_leaf_mj2_and_lacks_cert_for_output, ...
+                   @verify_single_tif_file_after_thawing, ...
+                   'do_submit', do_submit, ...
+                   'do_try', do_try, ...
+                   'submit_host_name', submit_host_name, ...
+                   'maximum_running_slot_count', maximum_running_slot_count, ...
+                   'slots_per_job', slots_per_job, ...
+                   'bsub_options', bsub_options, ...
+                   'predicate_extra_args', {input_folder_name, output_folder_name}, ...
+                   'batch_function_extra_args', {input_folder_name, output_folder_name}) ;
+end
diff --git a/check_certs_after_verifying_thawing.m b/check_certs_after_verifying_thawing.m
@@ -0,0 +1,10 @@
+function check_certs_after_verifying_thawing(input_root_folder_path, output_root_folder_path)
+    % Makes sure there's a .similar-tif-exists for each .mj2 file in
+    % input_root_folder_path.
+    find_and_verify(...
+        input_root_folder_path, ...
+        @is_a_mj2_in_a_folder_with_no_subfolders, ...
+        @check_single_cert_after_verifying_thawing, ...
+        input_root_folder_path, ...
+        output_root_folder_path) ;
+end
diff --git a/check_single_cert_after_verifying_thawing.m b/check_single_cert_after_verifying_thawing.m
@@ -0,0 +1,7 @@
+function result = check_single_cert_after_verifying_thawing(input_file_path, input_root_folder_path, output_root_folder_path)
+    input_relative_file_path = relpath(input_file_path, input_root_folder_path) ;
+    output_relative_file_path = replace_extension(input_relative_file_path, '.tif') ;
+    output_file_path = fullfile(output_root_folder_path, output_relative_file_path) ;
+    cert_file_path = horzcat(output_file_path, '.is-similar-to-mj2') ;    
+    result = logical( exist(cert_file_path, 'file') ) ;
+end
diff --git a/compare_similarity_measures.m b/compare_similarity_measures.m
diff --git a/copy_file_for_find_and_batch.m b/copy_file_for_find_and_batch.m
@@ -0,0 +1,8 @@
+function copy_file_for_find_and_batch(tif_side_file_path, tif_root_folder_name, mj2_root_folder_name, varargin)
+    relative_file_path_of_tif_side_file = relpath(tif_side_file_path, tif_root_folder_name) ;
+    relative_file_path_of_mj2_side_file = relative_file_path_of_tif_side_file ;
+    mj2_side_file_path = fullfile(mj2_root_folder_name, relative_file_path_of_mj2_side_file) ;
+    mj2_side_folder_path = fileparts2(mj2_side_file_path) ;
+    ensure_folder_exists(mj2_side_folder_path) ;
+    copyfile(tif_side_file_path, mj2_side_file_path) ;                   
+end
diff --git a/copy_non_mj2_files_to_tif_side.m b/copy_non_mj2_files_to_tif_side.m
@@ -0,0 +1,22 @@
+function copy_non_mj2_files_to_tif_side(output_folder_name, ...
+                                        input_folder_name)
+    % Find all non-.mj2 files, copy them over to the tif side
+
+    do_submit = false ;
+    do_try = true ;
+    maximum_running_slot_count = 400 ;  % doesn't matter
+    slots_per_job = 1 ;  % doesn't matter
+    bsub_options = '-P mouselight -W 59 -J copy' ;  % doesn't matter
+    submit_host_name = '' ;
+    find_and_batch(input_folder_name, ...
+                   @is_a_non_mj2_and_target_missing, ...
+                   @copy_file_for_find_and_batch, ...
+                   'do_submit', do_submit, ...
+                   'do_try', do_try, ...
+                   'submit_host_name', submit_host_name, ...
+                   'maximum_running_slot_count', maximum_running_slot_count, ...
+                   'slots_per_job', slots_per_job, ...
+                   'bsub_options', bsub_options, ...
+                   'predicate_extra_args', {input_folder_name, output_folder_name}, ...
+                   'batch_function_extra_args', {input_folder_name, output_folder_name}) ;
+end
diff --git a/copy_non_tif_files_to_mj2_side.m b/copy_non_tif_files_to_mj2_side.m
@@ -0,0 +1,22 @@
+function copy_non_tif_files_to_mj2_side(mj2_folder_name, ...
+                                        tif_folder_name)
+    % Find all non-.tif files, copy them over to the mj2 side
+
+    do_submit = false ;
+    do_try = true ;
+    maximum_running_slot_count = 400 ;  % doesn't matter
+    slots_per_job = 1 ;  % doesn't matter
+    bsub_options = '-P mouselight -W 59 -J copy' ;  % doesn't matter
+    submit_host_name = '' ;
+    find_and_batch(tif_folder_name, ...
+                   @is_a_non_tif_and_target_missing, ...
+                   @copy_file_for_find_and_batch, ...
+                   'do_submit', do_submit, ...
+                   'do_try', do_try, ...
+                   'submit_host_name', submit_host_name, ...
+                   'maximum_running_slot_count', maximum_running_slot_count, ...
+                   'slots_per_job', slots_per_job, ...
+                   'bsub_options', bsub_options, ...
+                   'predicate_extra_args', {tif_folder_name, mj2_folder_name} , ...
+                   'batch_function_extra_args', {tif_folder_name, mj2_folder_name} ) ;
+end
diff --git a/create_octree_test_folder.m b/create_octree_test_folder.m
@@ -0,0 +1,26 @@
+output_folder_name = '/groups/mousebrainmicro/mousebrainmicro/scripts/octree-freezing/octree-rebuilding-test-input-folder' ;
+input_folder_name = '/nrs/mouselight/SAMPLES/2023-02-23' ;
+max_depth = 2 ;
+do_submit = true ;
+do_try = false ;
+maximum_running_slot_count = 400 ;
+slots_per_job = 1 ;
+bsub_options = '-P mouselight -W 59 -J copy' ; 
+submit_host_name = 'login2.int.janelia.org' ;
+
+% Delete the test output folder
+reset_for_test(output_folder_name) ;
+
+% Do the copy
+find_and_batch(input_folder_name, ...
+               @(file_path)(true), ...
+               @copy_file_for_find_and_batch, ...
+               'do_submit', do_submit, ...
+               'do_try', do_try, ...
+               'submit_host_name', submit_host_name, ...
+               'maximum_running_slot_count', maximum_running_slot_count, ...
+               'slots_per_job', slots_per_job, ...
+               'bsub_options', bsub_options, ...
+               'max_depth', max_depth, ...
+               'predicate_extra_args', cell(1,0), ...
+               'batch_function_extra_args', {input_folder_name, output_folder_name}) ;
diff --git a/does_folder_have_subfolders.m b/does_folder_have_subfolders.m
@@ -0,0 +1,4 @@
+function result = does_folder_have_subfolders(folder_name)
+    [name_from_thing_index, is_folder_from_thing_index] = simple_dir(folder_name) ;  %#ok<ASGLU> 
+    result = any(is_folder_from_thing_index) ;
+end
diff --git a/does_need_preverification_after_freezing.m b/does_need_preverification_after_freezing.m
@@ -1,5 +1,4 @@
 function result = does_need_preverification_after_freezing(tif_file_path, tif_root_path, mj2_root_path)
-    [~,file_name] = fileparts2(tif_file_path) ;
-    result = does_file_name_end_in_dot_tif(file_name) && ...
+    result = is_a_tif_and_target_present(tif_file_path, tif_root_path, mj2_root_path) && ...
              ~does_is_similar_to_tif_check_file_exist(tif_file_path, tif_root_path, mj2_root_path) ;
 end
diff --git a/find_and_count.m b/find_and_count.m
@@ -0,0 +1,37 @@
+function result = find_and_count(base_folder_path, is_file_a_keeper_predicate, varargin)
+    result = ...
+        find_and_list_helper(base_folder_path, ...
+                             is_file_a_keeper_predicate, ...
+                             0, ...
+                             varargin{:}) ;
+end
+
+
+
+function result = ...
+        find_and_list_helper(base_folder_path, ...
+                             is_file_a_keeper_predicate, ...
+                             incoming_count, ...
+                             varargin)
+    count_so_far = incoming_count ;
+    [file_names, is_file_a_folder] = simple_dir(base_folder_path) ;
+    file_count = length(file_names) ;
+    for i = 1 : file_count ,
+        file_name = file_names{i} ;
+        is_this_file_a_folder = is_file_a_folder(i) ;
+        file_path = fullfile(base_folder_path, file_name) ;
+        if is_this_file_a_folder ,
+            % if a folder, recurse
+            count_so_far = ...
+                find_and_list_helper(file_path, ...
+                                     is_file_a_keeper_predicate, ...
+                                     count_so_far, ...
+                                     varargin{:}) ;
+        else
+            if feval(is_file_a_keeper_predicate, file_path, varargin{:}) ,
+                count_so_far = count_so_far + 1 ;
+            end
+        end
+    end    
+    result = count_so_far ;
+end
diff --git a/fraction_power_explained.m b/fraction_power_explained.m