Many improvements

batch_preverify_tif_files_after_mj2_from_tif.m

@@ -4,7 +4,7 @@ function batch_preverify_tif_files_after_mj2_from_tif(mj2_folder_name, tif_folde
 
     bsub_options = '-n4 -P mouselight -W 59 -J verify-frozen' ;
     find_and_batch(tif_folder_name, ...
-                   @does_file_name_end_in_dot_tif, ...
+                   @does_need_preverification_after_freezing, ...
                    @preverify_single_tif_file_after_mj2_from_tif, ...
                    do_submit, ...
                    bsub_options, ...

does_is_similar_to_tif_check_file_exist.m

@@ -0,0 +1,7 @@
+function result = does_is_similar_to_tif_check_file_exist(tif_file_path, tif_root_path, mj2_root_path)
+    tif_relative_path = relpath(tif_file_path, tif_root_path) ;
+    mj2_relative_path = replace_extension(tif_relative_path, '.mj2') ;
+    mj2_file_path = fullfile(mj2_root_path, mj2_relative_path) ;
+    check_file_path = horzcat(mj2_file_path, '.is-similar-to-tif') ;
+    result = logical(exist(check_file_path, 'file')) ;
+end

does_need_preverification_after_freezing.m

@@ -0,0 +1,5 @@
+function result = does_need_preverification_after_freezing(tif_file_path, tif_root_path, mj2_root_path)
+    [~,file_name] = fileparts2(tif_file_path) ;
+    result = does_file_name_end_in_dot_tif(file_name) && ...
+             ~does_is_similar_to_tif_check_file_exist(tif_file_path, tif_root_path, mj2_root_path) ;
+end

freeze_2018_10_01.m

@@ -0,0 +1,9 @@
+sample_date = '2018-10-01' ;
+mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ;
+compression_ratio = 10 ;
+do_verify = false ;
+do_run_on_cluster = true ;
+
+mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, compression_ratio, do_verify, do_run_on_cluster)
+is_all_well = verify_frozen_files(mj2_output_folder_path, tif_input_folder_path, do_run_on_cluster) 

freeze_2018_12_01.m

@@ -0,0 +1,9 @@
+sample_date = '2018-12-01' ;
+mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s/Tiling', sample_date) ;
+tif_input_folder_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_1_line_fix_output', sample_date) ;
+compression_ratio = 10 ;
+do_verify = false ;
+do_run_on_cluster = true ;
+
+mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, compression_ratio, do_verify, do_run_on_cluster)
+is_all_well = verify_frozen_files(mj2_output_folder_path, tif_input_folder_path, do_run_on_cluster) 

freeze_sample.m

@@ -0,0 +1,42 @@
+mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, compression_ratio, do_verify, do_run_on_cluster)
+%is_all_well = verify_frozen_files(mj2_output_folder_path, tif_input_folder_path, do_run_on_cluster) 
+
+fprintf('Doing batch preverification of .tif files...\n') ;
+batch_preverify_tif_files_after_mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, do_run_on_cluster)
+fprintf('Done with batch preverification of .tif files.\n') ;
+
+fprintf('Doing verification of .tif files...\n') ;
+verify_tif_files_after_mj2_from_tif(tif_input_folder_path, mj2_output_folder_path) ;
+fprintf('Done with verification of .tif files.\n') ;
+
+fprintf('Doing verification of non-.tif files...\n') ;        
+verify_non_tif_files_after_mj2_from_tif(mj2_output_folder_path, tif_input_folder_path) ;
+fprintf('Done with verification of non-.tif files.\n') ;        
+
+% Delete certificate files (just run locally)
+find_and_batch(mj2_output_folder_path, ...
+               @does_file_name_match_certificate_file, ...
+               @delete, ...
+               false, ...
+               '') ;
+
+% Do a final comparison of .tif files and .mj2 files, just for the heck of it
+[status, result] = system(sprintf('find %s -name "*.tif" -print | wc -l', tif_input_folder_path)) ;
+if status~= 0 ,
+    error('Unable to count number of .tif files\n') ;
+end
+final_tif_count = str2double(result) ;
+fprintf('.tif count in %s is: %d\n', tif_input_folder_path, final_tif_count) ;
+
+[status, result] = system(sprintf('find %s -name "*.mj2" -print | wc -l', mj2_output_folder_path)) ;
+if status~= 0 ,
+    error('Unable to count final number of .mj2 files\n') ;
+end
+final_mj2_count = str2double(result) ;
+fprintf('Final .mj2 count in %s is: %d\n', mj2_output_folder_path, final_mj2_count) ;
+
+% Report final status if we get this far
+fprintf('Compression and verification succeeded!\n') ;
+
+
+

mj2_from_tif.m

@@ -18,7 +18,7 @@ function mj2_from_tif(mj2_output_folder_name, tif_input_folder_name, compression
     job_ids = mj2_from_tif_helper(mj2_output_folder_name, tif_input_folder_name, compression_ratio, do_verify, do_run_on_cluster, zeros(1,0)) ;
 
     % Wait for the jobs to finish
-    fprintf('Waiting for mj2_from_tif() bjobs to finish...\n') ;
+    fprintf('Waiting for %d mj2_from_tif() bjobs to finish...\n', length(job_ids)) ;
     bwait(job_ids) ;
     fprintf('mj2_from_tif() bjobs are done.\n') ;
 
@@ -32,7 +32,7 @@ function mj2_from_tif(mj2_output_folder_name, tif_input_folder_name, compression
     if ~exist(mj2_output_folder_name, 'dir') ,
         mkdir(mj2_output_folder_name) ;
     end
-    tif_input_entity_names = setdiff(simple_dir(tif_input_folder_name), {'.' '..'}) ;
+    tif_input_entity_names = simple_dir(tif_input_folder_name) ;
     tif_input_entity_count = length(tif_input_entity_names) ;
     for i = 1 : tif_input_entity_count ,
         tif_input_entity_name = tif_input_entity_names{i} ;        

preverify_single_tif_file_after_mj2_from_tif.m

@@ -17,6 +17,10 @@ function preverify_single_tif_file_after_mj2_from_tif(tif_file_path, tif_root_fo
         tif_stack = read_16bit_grayscale_tif(tif_file_path) ;
         if is_mj2_similar_to_tif(mj2_stack, tif_stack) ,
             touch(check_file_path) ;
+        else
+            %nop() ;
         end
+    else
+        %nop() ;
     end
 end

test_freezing_locally.m

@@ -0,0 +1,13 @@
+path_to_this_script = mfilename('fullpath') ;
+path_to_this_folder = fileparts(path_to_this_script) ;
+mj2_output_folder_path = fullfile(path_to_this_folder, 'freezing-test-output-folder') ;
+tif_input_folder_path = fullfile(path_to_this_folder, 'freezing-test-input-folder') ;
+compression_ratio = 10 ;
+do_verify = false ;
+do_run_on_cluster = false ;
+
+% Delete the test output folder
+reset_for_test(mj2_output_folder_path) ;
+
+% Call the script
+freeze_sample

test_freezing_on_cluster.m

@@ -0,0 +1,13 @@
+path_to_this_script = mfilename('fullpath') ;
+path_to_this_folder = fileparts(path_to_this_script) ;
+mj2_output_folder_path = fullfile(path_to_this_folder, 'freezing-test-output-folder') ;
+tif_input_folder_path = fullfile(path_to_this_folder, 'freezing-test-input-folder') ;
+compression_ratio = 10 ;
+do_verify = false ;
+do_run_on_cluster = true ;
+
+% Delete the test output folder
+reset_for_test(mj2_output_folder_path) ;
+
+% Call the script
+freeze_sample

verify_non_tif_files_after_mj2_from_tif.m

@@ -1,11 +1,10 @@
-function is_all_well = verify_non_tif_files_after_mj2_from_tif(mj2_root_folder_path, tif_root_folder_path)
+function verify_non_tif_files_after_mj2_from_tif(mj2_root_folder_path, tif_root_folder_path)
     % Makes sure all the target files are present in the the destination, and have
     % the right size, or at least a plausible size in the case of .mj2's.
 
-    is_all_well = ...
-        find_and_verify(tif_root_folder_path, ...
-                        @does_file_name_not_end_in_dot_tif, ...
-                        @verify_single_non_tif_file_after_mj2_from_tif, ...
-                        tif_root_folder_path, ...
-                        mj2_root_folder_path) ;
+    find_and_verify(tif_root_folder_path, ...
+                    @does_file_name_not_end_in_dot_tif, ...
+                    @verify_single_non_tif_file_after_mj2_from_tif, ...
+                    tif_root_folder_path, ...
+                    mj2_root_folder_path) ;
 end

verify_tif_files_after_mj2_from_tif.m

@@ -1,10 +1,10 @@
-function is_all_well = verify_tif_files_after_mj2_from_tif(tif_root_folder_path, mj2_root_folder_path)
+function verify_tif_files_after_mj2_from_tif(tif_root_folder_path, mj2_root_folder_path)
     % Makes sure there's a .similar-mj2-exists for each .tif file in
     % tif_root_folder_path.    
-    is_all_well = find_and_verify(tif_root_folder_path, ...
-                                  @does_file_name_end_in_dot_tif, ...
-                                  @verify_single_tif_file_after_mj2_from_tif, ...
-                                  tif_root_folder_path, ...
-                                  mj2_root_folder_path) ;
+    find_and_verify(tif_root_folder_path, ...
+                    @does_file_name_end_in_dot_tif, ...
+                    @verify_single_tif_file_after_mj2_from_tif, ...
+                    tif_root_folder_path, ...
+                    mj2_root_folder_path) ;
 end