Using mouselight_toolbox more

MouseLightProject · Nov 3, 2019 · 5d86faf · 5d86faf
1 parent adedd6a
commit 5d86faf
Show file tree

Hide file tree

Showing 31 changed files with 88 additions and 244 deletions.
diff --git a/.gitignore b/.gitignore
@@ -8,3 +8,5 @@ mj2-from-tif-test-input-folder-virgin
 mj2-from-tif-test-input-folder
 mj2-from-tif-test-output-folder
 tif-from-mj2-test-input-folder
+freezing-test*
+thawing-test*
diff --git a/delete_job_array.m b/delete_job_array.m
diff --git a/dir_without_dot_and_dot_dot.m b/dir_without_dot_and_dot_dot.m
diff --git a/fif.m b/fif.m
diff --git a/file_size.m b/file_size.m
diff --git a/fraction_power_explained.m b/fraction_power_explained.m
@@ -14,7 +14,7 @@
             % 1 in this case because we divided by 2^16-1.  So PSNR = 1/MSE.  So the "PNSR"
             % is just MSE.  So 1-mse is the fraction of variance explained
 
-            result = 1 - error_power/image_power
+            result = 1 - error_power/image_power ;
         end
     else
         result = 0 ;

diff --git a/freeze_2018_07_02.m b/freeze_2018_07_02.m
@@ -4,4 +4,6 @@
 compression_ratio = 10 ;
 do_verify = false ;
 do_run_on_cluster = true ;
+
 mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, compression_ratio, do_verify, do_run_on_cluster)
+is_all_well = verify_frozen_files(mj2_output_folder_path, tif_input_folder_path, do_run_on_cluster) 
diff --git a/get_host_name.m b/get_host_name.m
diff --git a/get_scratch_folder_path.m b/get_scratch_folder_path.m
diff --git a/get_user_name.m b/get_user_name.m
diff --git a/is_mj2_similar_to_tif.m b/is_mj2_similar_to_tif.m
@@ -14,7 +14,7 @@
             % mean_summary_error = voxelwise_mean_relative_error_with_guard(mj2_stack, tif_stack, 1/256) ;
             % result = (mean_summary_error<0.1) ;  % this is just intended as a gross sanity check, so use a tolerant threshold
 
-            fpe_value = fraction_power_explained(mj2_stack, tif_stack) 
+            fpe_value = fraction_power_explained(mj2_stack, tif_stack) ;
             result = (fpe_value > 0.99) ;
         end
     else

diff --git a/mj2_from_tif.m b/mj2_from_tif.m
@@ -12,7 +12,23 @@ function mj2_from_tif(mj2_output_folder_name, tif_input_folder_name, compression
     if ~exist('do_run_on_cluster', 'var') || isempty(do_run_on_cluster) ,
         do_run_on_cluster = false ;
     end
+
+    % Call the helper
+    fprintf('Compressing .tifs to .mj2s...\n') ;
+    job_ids = mj2_from_tif_helper(mj2_output_folder_name, tif_input_folder_name, compression_ratio, do_verify, do_run_on_cluster, zeros(1,0)) ;
 
+    % Wait for the jobs to finish
+    fprintf('Waiting for mj2_from_tif() bjobs to finish...\n') ;
+    bwait(job_ids) ;
+    fprintf('mj2_from_tif() bjobs are done.\n') ;
+
+    fprintf('Done compressing .tifs to .mj2s.\n') ;    
+end
+
+
+
+function job_ids = mj2_from_tif_helper(mj2_output_folder_name, tif_input_folder_name, compression_ratio, do_verify, do_run_on_cluster, initial_job_ids)
+    job_ids = initial_job_ids ;
     if ~exist(mj2_output_folder_name, 'dir') ,
         mkdir(mj2_output_folder_name) ;
     end
@@ -24,53 +40,28 @@ function mj2_from_tif(mj2_output_folder_name, tif_input_folder_name, compression
         if exist(tif_input_entity_path, 'dir') ,
             % if a folder, recurse
             mj2_output_entity_path = fullfile(mj2_output_folder_name, tif_input_entity_name) ;
-            mj2_from_tif(mj2_output_entity_path, tif_input_entity_path, compression_ratio, do_verify, do_run_on_cluster) ;
+            job_ids = mj2_from_tif_helper(mj2_output_entity_path, tif_input_entity_path, compression_ratio, do_verify, do_run_on_cluster, job_ids) ;
         else
             % if a normal file, convert to .mj2 if it's a .tif, or just
             % copy otherwise
             [~,~,ext] = fileparts(tif_input_entity_path) ;
             if isequal(ext, '.tif') ,
                 mj2_output_entity_path = fullfile(mj2_output_folder_name, replace_extension(tif_input_entity_name, '.mj2')) ;
                 if ~exist(mj2_output_entity_path, 'file') ,
-                    fprintf('Converting %s...\n', tif_input_entity_path);
-                    tic_id = tic() ;
-                    scratch_folder_path = get_scratch_folder_path() ;
-                    temporary_tif_input_file_path = [tempname(scratch_folder_path) '.tif'] ;
-                    temporary_mj2_output_file_path = [tempname(scratch_folder_path) '.mj2'] ;  % risky, but VideoWriter is dumb and always adds this anyway                   
-                    if do_run_on_cluster ,                        
-                        command_line = ...
-                            sprintf(['matlab -nojvm ' ...
-                                     '-batch ' ...
-                                         '"mj2_output_entity_path = ''%s'';  tif_input_entity_path = ''%s''; do_verify = %s; ' ...
-                                         'compression_ratio = %s; ' ...
-                                         'temporary_tif_input_file_path = ''%s'';  temporary_mj2_output_file_path = ''%s''; ' ...                                     
-                                         'copyfile(tif_input_entity_path, temporary_tif_input_file_path); ' ...
-                                         'mj2_from_tif_single(temporary_mj2_output_file_path, temporary_tif_input_file_path, compression_ratio, do_verify); ' ...
-                                         'copyfile(temporary_mj2_output_file_path, mj2_output_entity_path); ' ...
-                                         'delete(temporary_tif_input_file_path); ' ...
-                                         'delete(temporary_mj2_output_file_path);"'], ...
-                                    mj2_output_entity_path, ...
-                                    tif_input_entity_path, ...
-                                    fif(do_verify, 'true', 'false'),  ...                                    
-                                    num2str(compression_ratio, '%.18g') , ...
-                                    temporary_tif_input_file_path, ...
-                                    temporary_mj2_output_file_path) ;
-                        fprintf('Command line: %s\n', command_line) ;
-                        if do_verify , 
-                            core_count_to_request = 4 ;
-                        else
-                            core_count_to_request = 1 ;
-                        end
-                        bsub_command_line = ...
-                            sprintf('bsub -n%d -P mouselight -oo /dev/null -eo /dev/null -W 59 -J freeze %s', core_count_to_request, command_line) ;
-                        %bsub_command_line = sprintf('bsub -n%d -P mouselight %s', core_count_to_request, command_line) ;
-                        fprintf('bsub Command line: %s\n', bsub_command_line) ;
-                        system(bsub_command_line) ;
+                    if do_verify , 
+                        core_count_to_request = 4 ;
                     else
-                        mj2_from_tif_single(mj2_output_entity_path, tif_input_entity_path, compression_ratio, do_verify) ;
+                        core_count_to_request = 1 ;
                     end
-                    duration  = toc(tic_id) ;
-                    fprintf('Took %g seconds\n', duration) ;
+                    bsub_options = sprintf('-n%d -P mouselight -W 59 -J freeze', core_count_to_request) ;
+                    job_id = bsub(do_run_on_cluster, ...
+                                  bsub_options, ...
+                                  @mj2_from_tif_single_with_temps, ...
+                                      mj2_output_entity_path, ...
+                                      tif_input_entity_path, ...
+                                      compression_ratio, ...
+                                      do_verify) ;
+                    job_ids = horzcat(job_ids, job_id) ;  %#ok<AGROW>
                 end
             else
                 mj2_output_entity_path = fullfile(mj2_output_folder_name, tif_input_entity_name) ;  % input not really a tif, output not really a .mj2

diff --git a/mj2_from_tif_single.m b/mj2_from_tif_single.m
@@ -28,12 +28,12 @@ function mj2_from_tif_single(mj2_output_file_name, tif_input_file_name, compress
     % without error.
 
     if do_verify ,
-        ticId = tic() ;
+        %ticId = tic() ;
         mj2_stack = read_16bit_grayscale_mj2(mj2_output_file_name) ;
         if ~is_mj2_similar_to_tif(mj2_stack, tif_stack) ,            
             error('%s is not sufficiently similar to %s\n', mj2_output_file_name, tif_input_file_name) ;
         end
-        elapsed_time = toc(ticId) ;
-        fprintf('Time to verify was %g seconds.\n', elapsed_time) ;
+        %elapsed_time = toc(ticId) ;
+        %fprintf('Time to verify was %g seconds.\n', elapsed_time) ;
     end
 end
diff --git a/mj2_from_tif_single_with_temps.m b/mj2_from_tif_single_with_temps.m
@@ -0,0 +1,10 @@
+function mj2_from_tif_single_with_temps(mj2_output_file_path, tif_input_file_path, compression_ratio, do_verify)
+    scratch_folder_path = get_scratch_folder_path() ;
+    temporary_tif_input_file_path = [tempname(scratch_folder_path) '.tif'] ;
+    temporary_mj2_output_file_path = [tempname(scratch_folder_path) '.mj2'] ;  % risky, but VideoWriter is dumb and always adds this anyway
+    copyfile(tif_input_file_path, temporary_tif_input_file_path);
+    mj2_from_tif_single(temporary_mj2_output_file_path, temporary_tif_input_file_path, compression_ratio, do_verify);
+    copyfile(temporary_mj2_output_file_path, mj2_output_file_path);
+    delete(temporary_tif_input_file_path);
+    delete(temporary_mj2_output_file_path);
+end
diff --git a/mouselight_toolbox b/mouselight_toolbox
diff --git a/preverify_single_file_after_mj2_from_tif.m b/preverify_single_file_after_mj2_from_tif.m
@@ -1,9 +1,10 @@
 function preverify_single_file_after_mj2_from_tif(tif_file_path, tif_root_folder_name, mj2_root_folder_name)
     % Checks that the mj2 is similar to the .tif, and outputs a .similar-mj2-exists file in the
     % tif folder if the are similar.  If unable to verify, writes nothing.
-    relative_file_path = relpath(tif_file_path, tif_root_folder_name) ;
-    mj2_file_path = fullfile(mj2_root_folder_name, relative_file_path) ;
-    check_file_path = horzcat(tif_file_path, '.similar-mj2-exists') ;    
+    relative_file_path_of_tif = relpath(tif_file_path, tif_root_folder_name) ;
+    relative_file_path_of_mj2 = replace_extension(relative_file_path_of_tif, '.mj2') ;
+    mj2_file_path = fullfile(mj2_root_folder_name, relative_file_path_of_mj2) ;
+    check_file_path = horzcat(mj2_file_path, '.is-similar-to-tif') ;    
     if exist(check_file_path, 'file') ,
         return
     end

diff --git a/read_16bit_grayscale_mj2.m b/read_16bit_grayscale_mj2.m
diff --git a/read_16bit_grayscale_tif.m b/read_16bit_grayscale_tif.m
diff --git a/read_file_into_uint8_array.m b/read_file_into_uint8_array.m
diff --git a/replace_extension.m b/replace_extension.m
diff --git a/reset_for_test.m b/reset_for_test.m
@@ -0,0 +1,3 @@
+function reset_for_test(output_folder_path)
+    system(sprintf('rm -rf %s', output_folder_path)) ;
+end
diff --git a/results_from_job_array.m b/results_from_job_array.m
diff --git a/simple_dir.m b/simple_dir.m
diff --git a/test_freezing.m b/test_freezing.m
@@ -1,8 +1,11 @@
 path_to_this_script = mfilename('fullpath') ;
 path_to_this_folder = fileparts(path_to_this_script) ;
-mj2_output_folder_path = fullfile(path_to_this_folder, 'mj2-from-tif-test-output-folder') ;
-tif_input_folder_path = fullfile(path_to_this_folder, 'mj2-from-tif-test-input-folder') ;
+mj2_output_folder_path = fullfile(path_to_this_folder, 'freezing-test-output-folder') ;
+tif_input_folder_path = fullfile(path_to_this_folder, 'freezing-test-input-folder') ;
 compression_ratio = 10 ;
-do_verify = true ;
-do_run_on_cluster = false ;
+do_verify = false ;
+do_run_on_cluster = true ;
+
+reset_for_test(mj2_output_folder_path) ;
 mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, compression_ratio, do_verify, do_run_on_cluster)
+is_all_well = verify_frozen_files(mj2_output_folder_path, tif_input_folder_path, do_run_on_cluster) 
diff --git a/touch.m b/touch.m
diff --git a/verify_big_files_after_mj2_from_tif.m b/verify_big_files_after_mj2_from_tif.m
@@ -1,6 +1,6 @@
-function is_all_well = verify_big_files_after_mj2_from_tif(tif_root_folder_path)
+function is_all_well = verify_big_files_after_mj2_from_tif(tif_root_folder_path, mj2_root_folder_path)
     % Makes sure there's a .similar-mj2-exists for each .tif file in
     % tif_root_folder_path.    
-    is_all_well = find_and_verify(tif_root_folder_path, '^\d\d\d\d\d-ngc.\d.tif$', @verify_single_big_file_after_mj2_from_tif) ;
+    is_all_well = find_and_verify(tif_root_folder_path, @does_match_raw_tile_tif_name, @verify_single_big_file_after_mj2_from_tif, tif_root_folder_path, mj2_root_folder_path) ;
 end
 
diff --git a/verify_file_sizes_after_mj2_from_tif.m b/verify_file_sizes_after_mj2_from_tif.m
@@ -30,7 +30,7 @@ function verify_file_sizes_after_mj2_from_tif(mj2_output_folder_name, tif_input_
                 mj2_output_entity_path = fullfile(mj2_output_folder_name, replace_extension(tif_input_entity_name, '.mj2')) ;
                 if exist(mj2_output_entity_path, 'file') ,
                     tif_file_size = tif_input_entity.bytes ;
-                    mj2_file_size = file_size(mj2_output_entity_path) ;
+                    mj2_file_size = get_file_size(mj2_output_entity_path) ;
                     desired_mj2_file_size = round(tif_file_size/compression_ratio) ;
                     if mj2_file_size == desired_mj2_file_size ,
                         % do nothing, this is fine and good, and should handle zero-length files
@@ -53,7 +53,7 @@ function verify_file_sizes_after_mj2_from_tif(mj2_output_folder_name, tif_input_
                 mj2_output_entity_path = fullfile(mj2_output_folder_name, tif_input_entity_name) ;  % input not really a tif, output not really a .mj2
                 if exist(mj2_output_entity_path, 'file') ,
                     tif_file_size = tif_input_entity.bytes ;
-                    mj2_file_size = file_size(mj2_output_entity_path) ;
+                    mj2_file_size = get_file_size(mj2_output_entity_path) ;
                     if mj2_file_size == tif_file_size ,
                         % do nothing, this is fine and good
                     else
+2 −2		find_and_batch.m
+14 −0		get_file_size.m
+24 −0		get_host_name.m
+12 −0		get_scratch_folder_path.m
+19 −0		get_user_name.m
+15 −0		read_16bit_grayscale_mj2.m
+8 −0		read_file_into_uint8_array.m
+12 −0		write_16bit_grayscale_mj2.m