Added code for 'feezing' samples also

.gitignore

@@ -4,3 +4,7 @@ aside
 mj2tif-test-input-folder
 mj2tif-test-output-folder
 repo
+mj2-from-tif-test-input-folder-virgin
+mj2-from-tif-test-input-folder
+mj2-from-tif-test-output-folder
+tif-from-mj2-test-input-folder

freeze_2018_12_01_sample.m

@@ -0,0 +1,3 @@
+do_verify = false ;
+do_on_cluster = true ;
+tif_from_mj2('/groups/mousebrainmicro/mousebrainmicro/data/2017-10-31-thawed', '/nrs/mouselight/SAMPLES/compressed_recovered/2017-10-31', do_verify, do_on_cluster) ;

is_mj2_similar_to_tif.m

@@ -0,0 +1,24 @@
+function result = is_mj2_similar_to_tif(mj2_file_name, tif_file_name)
+    % Checks if an mj2 file is similar to a .tif file.
+
+    mj2_stack = read_16bit_grayscale_mj2(mj2_file_name) ;
+    tif_stack = read_16bit_grayscale_tif(tif_file_name) ;
+    if isequal(class(mj2_stack), class(tif_stack)) && isequal(size(mj2_stack), size(tif_stack)) ,
+        if all(all(all(mj2_stack==tif_stack))) ,
+            result = true ;
+        else
+            ssim_value = ssim(mj2_stack, tif_stack) ;
+            result = (ssim_value>0.95) ;
+            if ~result ,
+                keyboard 
+            end
+%            mj2_stack_serial = double(mj2_stack(:)) ;
+%            tif_stack_serial = double(tif_stack(:)) ;            
+%             Rho_hat = corrcoef(mj2_stack_serial, tif_stack_serial) ;
+%             rho_hat = Rho_hat(1,2) 
+%             result = (rho_hat>0.9) ;
+        end
+    else
+        result = false ;
+    end
+end

mj2_from_tif.m

@@ -0,0 +1,100 @@
+function mj2_from_tif(mj2_output_folder_name, tif_input_folder_name, compression_ratio, do_verify, do_delete_input_files, do_run_on_cluster)
+    % Converts each .mj2 file in input_folder_name to a multi-image .tif in
+    % output_folder_name.  Will overwrite pre-existing files in
+    % output_folder_name, if present.
+
+    if ~exist('compression_ratio', 'var') || isempty(compression_ratio) ,
+        compression_ratio = 10 ;
+    end
+    if ~exist('do_verify', 'var') || isempty(do_verify) ,
+        do_verify = false ;
+    end
+    if ~exist('do_delete_input_files', 'var') || isempty(do_delete_input_files) ,
+        do_delete_input_files = false ;
+    end
+    if ~exist('do_run_on_cluster', 'var') || isempty(do_run_on_cluster) ,
+        do_run_on_cluster = false ;
+    end
+
+    if ~exist(mj2_output_folder_name, 'dir') ,
+        mkdir(mj2_output_folder_name) ;
+    end
+    tif_input_entity_names = setdiff(simple_dir(tif_input_folder_name), {'.' '..'}) ;
+    tif_input_entity_count = length(tif_input_entity_names) ;
+    for i = 1 : tif_input_entity_count ,
+        tif_input_entity_name = tif_input_entity_names{i} ;        
+        tif_input_entity_path = fullfile(tif_input_folder_name, tif_input_entity_name) ;
+        if exist(tif_input_entity_path, 'dir') ,
+            % if a folder, recurse
+            mj2_output_entity_path = fullfile(mj2_output_folder_name, tif_input_entity_name) ;
+            mj2_from_tif(mj2_output_entity_path, tif_input_entity_path, compression_ratio, do_verify, do_delete_input_files, do_run_on_cluster) ;
+        else
+            % if a normal file, convert to .mj2 if it's a .tif, or just
+            % copy otherwise
+            [~,~,ext] = fileparts(tif_input_entity_path) ;
+            if isequal(ext, '.tif') ,
+                mj2_output_entity_path = fullfile(mj2_output_folder_name, replace_extension(tif_input_entity_name, '.mj2')) ;
+                if ~exist(mj2_output_entity_path, 'file') ,
+                    fprintf('Converting %s...\n', tif_input_entity_path);
+                    tic_id = tic() ;
+                    scratch_folder_path = get_scratch_folder_path() ;
+                    temporary_tif_input_file_path = [tempname(scratch_folder_path) '.tif'] ;
+                    temporary_mj2_output_file_path = [tempname(scratch_folder_path) '.mj2'] ;  % risky, but VideoWriter is dumb and always adds this anyway                   
+                    if do_run_on_cluster ,                        
+                        command_line = ...
+                            sprintf(['matlab -nojvm ' ...
+                                     '-r "mj2_output_entity_path = ''%s'';  tif_input_entity_path = ''%s''; do_verify = %s; ' ...
+                                         'compression_ratio = %s; do_delete_input_file = %s; ' ...
+                                         'temporary_tif_input_file_path = ''%s'';  temporary_mj2_output_file_path = ''%s''; ' ...                                     
+                                         'copyfile(tif_input_entity_path, temporary_tif_input_file_path); ' ...
+                                         'mj2_from_tif_single(temporary_mj2_output_file_path, temporary_tif_input_file_path, compression_ratio, do_verify); ' ...
+                                         'copyfile(temporary_mj2_output_file_path, mj2_output_entity_path); ' ...
+                                         'delete(temporary_tif_input_file_path); ' ...
+                                         'delete(temporary_mj2_output_file_path); ' ...
+                                         'if do_delete_input_file, delete(tif_input_entity_path); end;  ' ...                                         
+                                         'exit"'], ...
+                                    mj2_output_entity_path, ...
+                                    tif_input_entity_path, ...
+                                    fif(do_verify, 'true', 'false'),  ...                                    
+                                    num2str(compression_ratio, '%.18g') , ...
+                                    fif(do_delete_input_files, 'true', 'false'), ...
+                                    temporary_tif_input_file_path, ...
+                                    temporary_mj2_output_file_path) ;
+                        fprintf('Command line: %s\n', command_line) ;                        
+                        %bsub_command_line = sprintf('bsub -P mouselight -o /dev/null -e /dev/null %s', command_line) ;
+                        if do_verify , 
+                            core_count_to_request = 4 ;
+                        else
+                            core_count_to_request = 1 ;
+                        end
+                        bsub_command_line = sprintf('bsub -n%d -P mouselight %s', core_count_to_request, command_line) ;
+                        fprintf('bsub Command line: %s\n', bsub_command_line) ;
+                        system(bsub_command_line) ;
+                    else
+                        mj2_from_tif_single(mj2_output_entity_path, tif_input_entity_path, compression_ratio, do_verify, do_delete_input_files) ;
+                    end
+                    duration  = toc(tic_id) ;
+                    fprintf('Took %g seconds\n', duration) ;
+                end
+            else
+                mj2_output_entity_path = fullfile(mj2_output_folder_name, tif_input_entity_name) ;  % input not really a tif, output not really a .mj2
+                if ~exist(mj2_output_entity_path, 'file') ,
+                    copyfile(tif_input_entity_path, mj2_output_entity_path) ;                    
+                end
+                % Delete input file, if desired
+                if do_delete_input_files ,
+                    delete(tif_input_entity_path) ;
+                end                
+            end
+        end
+    end
+
+    % Delete the input folder if we were asked to delete input files, and if the input folder is really empty
+    if do_delete_input_files ,
+        tif_input_entity_names = setdiff(simple_dir(tif_input_folder_name), {'.' '..'}) ;
+        tif_input_entity_count = length(tif_input_entity_names) ;
+        if tif_input_entity_count==0 ,
+            rmdir(tif_input_folder_name) ;
+        end
+    end
+end

mj2_from_tif_single.m

@@ -0,0 +1,37 @@
+function mj2_from_tif_single(mj2_output_file_name, tif_input_file_name, compression_ratio, do_verify, do_delete_input_file)
+    % Converts a single .tif file at input_file_name to a MJPEG-2000 file at
+    % output_file_name, using the given compression ratio.  Will overwrite
+    % pre-existing file at output_file_name, if present.
+
+    mj2_output_file_name
+    tif_input_file_name
+
+    % Process input args
+    if ~exist('compression_ratio', 'var')  || isempty(compression_ratio) ,
+        compression_ratio = 10 ;
+    end    
+    if ~exist('do_verify', 'var')  || isempty(do_verify) ,
+        do_verify = false ;
+    end    
+    if ~exist('do_delete_input_file', 'var')  || isempty(do_delete_input_file) ,
+        do_delete_input_file = false ;
+    end
+
+    % Read the input file
+    stack = read_16bit_grayscale_tif(tif_input_file_name) ;
+
+    % Write the output file    
+    write_16bit_grayscale_mj2(mj2_output_file_name, stack, compression_ratio) ;
+
+    % Verify, if desired
+    if do_verify ,
+        if ~is_mj2_similar_to_tif(mj2_output_file_name, tif_input_file_name) ,
+            error('%s is not sufficiently similar to %s\n', mj2_output_file_name, tif_input_file_name) ;
+        end
+    end
+
+    % Delete input file, if desired
+    if do_delete_input_file ,
+        delete(tif_input_file_name) ;
+    end    
+end

modpath.m

@@ -0,0 +1,2 @@
+function modpath()
+end

test_freezing.m

@@ -0,0 +1,9 @@
+path_to_this_script = mfilename('fullpath') ;
+path_to_this_folder = fileparts(path_to_this_script) ;
+mj2_output_folder_path = fullfile(path_to_this_folder, 'mj2-from-tif-test-output-folder') ;
+tif_input_folder_path = fullfile(path_to_this_folder, 'mj2-from-tif-test-input-folder') ;
+compression_ratio = 10 ;
+do_verify = true ;
+do_delete_input_files = true ;
+do_run_on_cluster = true ;
+mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, compression_ratio, do_verify, do_delete_input_files, do_run_on_cluster)

thaw_2017_10_31_sample.m

@@ -1,3 +1,3 @@
 do_verify = false ;
 do_on_cluster = true ;
-mj2tif('/nrs/mouselight/SAMPLES/compressed_recovered/2017-10-31', '/groups/mousebrainmicro/mousebrainmicro/data/2017-10-31-thawed', do_verify, do_on_cluster) ;
+tif_from_mj2('/groups/mousebrainmicro/mousebrainmicro/data/2017-10-31-thawed', '/nrs/mouselight/SAMPLES/compressed_recovered/2017-10-31', do_verify, do_on_cluster) ;

thaw_2017_12_19_sample.m

@@ -1,3 +1,3 @@
 do_verify = false ;
 do_on_cluster = true ;
-mj2tif('/groups/mousebrainmicro/mousebrainmicro/data/2017-12-19', '/groups/mousebrainmicro/mousebrainmicro/data/2017-12-19-thawed', do_verify, do_on_cluster) ;
+tif_from_mj2('/groups/mousebrainmicro/mousebrainmicro/data/2017-12-19-thawed', '/groups/mousebrainmicro/mousebrainmicro/data/2017-12-19', do_verify, do_on_cluster) ;

mj2tif.m → tif_from_mj2.m

@@ -1,4 +1,4 @@
-function mj2tif(input_folder_name, output_folder_name, do_verify, do_run_on_cluster)
+function tif_from_mj2(tif_output_folder_name, mj2_input_folder_name, do_verify, do_run_on_cluster)
     % Converts each .mj2 file in input_folder_name to a multi-image .tif in
     % output_folder_name.  Will overwrite pre-existing files in
     % output_folder_name, if present.
@@ -10,43 +10,43 @@ function mj2tif(input_folder_name, output_folder_name, do_verify, do_run_on_clus
         do_run_on_cluster = false ;
     end
 
-    if ~exist(output_folder_name, 'dir') ,
-        mkdir(output_folder_name) ;
+    if ~exist(tif_output_folder_name, 'dir') ,
+        mkdir(tif_output_folder_name) ;
     end
-    entity_names = setdiff(simple_dir(input_folder_name), {'.' '..'}) ;
+    entity_names = setdiff(simple_dir(mj2_input_folder_name), {'.' '..'}) ;
     entity_count = length(entity_names) ;
     for i = 1 : entity_count ,
-        entity_name = entity_names{i} ;        
-        entity_path = fullfile(input_folder_name, entity_name) ;
-        if exist(entity_path, 'dir') ,
+        mj2_input_entity_name = entity_names{i} ;        
+        mj2_input_entity_path = fullfile(mj2_input_folder_name, mj2_input_entity_name) ;
+        if exist(mj2_input_entity_path, 'dir') ,
             % if a folder, recurse
-            output_entity_path = fullfile(output_folder_name, entity_name) ;
-            mj2tif(entity_path, output_entity_path, do_verify, do_run_on_cluster) ;
+            tif_output_entity_path = fullfile(tif_output_folder_name, mj2_input_entity_name) ;
+            tif_from_mj2(tif_output_entity_path, mj2_input_entity_path, do_verify, do_run_on_cluster) ;
         else
             % if a normal file, convert to .tif if it's a .mj2, or just
             % copy otherwise
-            [~,~,ext] = fileparts(entity_path) ;
+            [~,~,ext] = fileparts(mj2_input_entity_path) ;
             if isequal(ext, '.mj2') ,
-                output_entity_path = fullfile(output_folder_name, replace_extension(entity_name, '.tif')) ;
-                if ~exist(output_entity_path, 'file') ,
-                    fprintf('Converting %s...\n', entity_path);
+                tif_output_entity_path = fullfile(tif_output_folder_name, replace_extension(mj2_input_entity_name, '.tif')) ;
+                if ~exist(tif_output_entity_path, 'file') ,
+                    fprintf('Converting %s...\n', mj2_input_entity_path);
                     tic_id = tic() ;
                     scratch_folder_path = get_scratch_folder_path() ;
-                    temporary_input_file_path = tempname(scratch_folder_path)
-                    temporary_output_file_path = tempname(scratch_folder_path)                    
+                    temporary_input_file_path = tempname(scratch_folder_path) ;
+                    temporary_output_file_path = tempname(scratch_folder_path) ;                   
                     if do_run_on_cluster ,                        
                         command_line = ...
                             sprintf(['matlab -nojvm -singleCompThread ' ...
-                                     '-r "entity_path = ''%s'';  output_entity_path = ''%s''; do_verify = %s; ' ...
+                                     '-r "mj2_entity_path = ''%s'';  tif_output_entity_path = ''%s''; do_verify = %s; ' ...
                                          'temporary_input_file_path = ''%s'';  temporary_output_file_path = ''%s''; ' ...                                     
-                                         'copyfile(entity_path, temporary_input_file_path); ' ...
-                                         'mj2tif_single(temporary_input_file_path, temporary_output_file_path, do_verify); ' ...
-                                         'copyfile(temporary_output_file_path, output_entity_path); ' ...
+                                         'copyfile(mj2_entity_path, temporary_input_file_path); ' ...
+                                         'tif_from_mj2_single(temporary_output_file_path, temporary_input_file_path, do_verify); ' ...
+                                         'copyfile(temporary_output_file_path, tif_output_entity_path); ' ...
                                          'delete(temporary_input_file_path); ' ...
                                          'delete(temporary_output_file_path); ' ...
                                          'exit"'], ...
-                                    entity_path, ...
-                                    output_entity_path, ...
+                                    mj2_input_entity_path, ...
+                                    tif_output_entity_path, ...
                                     fif(do_verify, 'true', 'false'),  ...
                                     temporary_input_file_path, ...
                                     temporary_output_file_path) ;
@@ -55,15 +55,15 @@ function mj2tif(input_folder_name, output_folder_name, do_verify, do_run_on_clus
                         fprintf('bsub Command line: %s\n', bsub_command_line) ;
                         system(bsub_command_line) ;
                     else
-                        mj2tif_single(entity_path, output_entity_path, do_verify) ;
+                        tif_from_mj2_single(tif_output_entity_path, mj2_input_entity_path, do_verify) ;
                     end
                     duration  = toc(tic_id) ;
                     fprintf('Took %g seconds\n', duration) ;
                 end
             else
-                output_entity_path = fullfile(output_folder_name, entity_name) ;
-                if ~exist(output_entity_path, 'file') ,
-                    copyfile(entity_path, output_entity_path) ;
+                tif_output_entity_path = fullfile(tif_output_folder_name, mj2_input_entity_name) ;
+                if ~exist(tif_output_entity_path, 'file') ,
+                    copyfile(mj2_input_entity_path, tif_output_entity_path) ;
                 end
             end
         end

tif_from_mj2_single.m

@@ -0,0 +1,27 @@
+function tif_from_mj2_single(tif_output_file_name, mj2_input_file_name, do_verify)
+    % Converts a single .mj2 file at input_file_name to a multi-image .tif
+    % at output_file_name.  Will overwrite pre-existing file at
+    % output_file_name, if present.
+
+    if ~exist('do_verify', 'var') || isempty(do_verify) ,
+        do_verify = false ;
+    end
+
+    input_file = VideoReader(mj2_input_file_name) ;
+    frame_index = 0 ;
+    while input_file.hasFrame() ,
+        frame_index = frame_index + 1 ; 
+        frame = input_file.readFrame() ;
+        if frame_index == 1 ,
+            imwrite(frame, tif_output_file_name, 'tif', 'WriteMode', 'overwrite',  'Compression', 'deflate') ;
+        else
+            imwrite(frame, tif_output_file_name, 'tif', 'WriteMode', 'append',  'Compression', 'deflate') ;            
+        end        
+    end    
+
+    if do_verify ,
+        if ~is_mj2_same_as_tif(mj2_input_file_name, tif_output_file_name) ,
+            error('%s is not the same as %s\n', tif_output_file_name, mj2_input_file_name) ;
+        end
+    end
+end

write_16bit_grayscale_mj2.m

@@ -0,0 +1,12 @@
+function write_16bit_grayscale_mj2(file_name, stack, compression_ratio) 
+    fprintf('Nominal file name in write_16bit_grayscale_mj2(): %s\n', file_name) ;
+    [row_count, column_count, page_count] = size(stack) ;
+    stack_for_video_writer = reshape(stack, [row_count, column_count 1 page_count]) ;
+      % VideoWriter supports RGB, so have time in 4th dimension to disambiguate.
+    vw = VideoWriter(file_name, 'Motion JPEG 2000') ;
+    vw.CompressionRatio = compression_ratio ;
+    vw.MJ2BitDepth = 16 ;
+    vw.open() ;    
+    vw.writeVideo(stack_for_video_writer) ;
+    vw.close() ;
+end

write_16bit_grayscalr_mj2.m

@@ -0,0 +1,8 @@
+function write_16bit_grayscalr_mj2(file_name, stack) 
+    vw = VideoWriter(file_name, 'Motion JPEG 2000') ;
+    vw.CompressionRatio = compression_ratio ;
+    vw.MJ2BitDepth = 16 ;
+    vw.open() ;
+    vw.writeVideo(stack) ;
+    vw.close() ;
+end