From 64fdb80632d903476dc79e230ccc08bcda25fef9 Mon Sep 17 00:00:00 2001 From: "Adam L. Taylor" Date: Thu, 10 Oct 2019 18:47:41 -0400 Subject: [PATCH] Added code for 'feezing' samples also --- .gitignore | 4 ++ freeze_2018_12_01_sample.m | 3 ++ is_mj2_similar_to_tif.m | 24 +++++++++ mj2_from_tif.m | 100 ++++++++++++++++++++++++++++++++++++ mj2_from_tif_single.m | 37 +++++++++++++ mj2tif_single.m | 27 ---------- modpath.m | 2 + test_freezing.m | 9 ++++ thaw_2017_10_31_sample.m | 2 +- thaw_2017_12_19_sample.m | 2 +- mj2tif.m => tif_from_mj2.m | 50 +++++++++--------- tif_from_mj2_single.m | 27 ++++++++++ write_16bit_grayscale_mj2.m | 12 +++++ write_16bit_grayscalr_mj2.m | 8 +++ 14 files changed, 253 insertions(+), 54 deletions(-) create mode 100644 freeze_2018_12_01_sample.m create mode 100644 is_mj2_similar_to_tif.m create mode 100644 mj2_from_tif.m create mode 100644 mj2_from_tif_single.m delete mode 100644 mj2tif_single.m create mode 100644 modpath.m create mode 100644 test_freezing.m rename mj2tif.m => tif_from_mj2.m (54%) create mode 100644 tif_from_mj2_single.m create mode 100644 write_16bit_grayscale_mj2.m create mode 100644 write_16bit_grayscalr_mj2.m diff --git a/.gitignore b/.gitignore index b211b9d..ce58cf8 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ aside mj2tif-test-input-folder mj2tif-test-output-folder repo +mj2-from-tif-test-input-folder-virgin +mj2-from-tif-test-input-folder +mj2-from-tif-test-output-folder +tif-from-mj2-test-input-folder diff --git a/freeze_2018_12_01_sample.m b/freeze_2018_12_01_sample.m new file mode 100644 index 0000000..c7cb86d --- /dev/null +++ b/freeze_2018_12_01_sample.m @@ -0,0 +1,3 @@ +do_verify = false ; +do_on_cluster = true ; +tif_from_mj2('/groups/mousebrainmicro/mousebrainmicro/data/2017-10-31-thawed', '/nrs/mouselight/SAMPLES/compressed_recovered/2017-10-31', do_verify, do_on_cluster) ; diff --git a/is_mj2_similar_to_tif.m b/is_mj2_similar_to_tif.m new file mode 100644 index 0000000..4d1df15 --- /dev/null +++ b/is_mj2_similar_to_tif.m @@ -0,0 +1,24 @@ +function result = is_mj2_similar_to_tif(mj2_file_name, tif_file_name) + % Checks if an mj2 file is similar to a .tif file. + + mj2_stack = read_16bit_grayscale_mj2(mj2_file_name) ; + tif_stack = read_16bit_grayscale_tif(tif_file_name) ; + if isequal(class(mj2_stack), class(tif_stack)) && isequal(size(mj2_stack), size(tif_stack)) , + if all(all(all(mj2_stack==tif_stack))) , + result = true ; + else + ssim_value = ssim(mj2_stack, tif_stack) ; + result = (ssim_value>0.95) ; + if ~result , + keyboard + end +% mj2_stack_serial = double(mj2_stack(:)) ; +% tif_stack_serial = double(tif_stack(:)) ; +% Rho_hat = corrcoef(mj2_stack_serial, tif_stack_serial) ; +% rho_hat = Rho_hat(1,2) +% result = (rho_hat>0.9) ; + end + else + result = false ; + end +end diff --git a/mj2_from_tif.m b/mj2_from_tif.m new file mode 100644 index 0000000..adb3862 --- /dev/null +++ b/mj2_from_tif.m @@ -0,0 +1,100 @@ +function mj2_from_tif(mj2_output_folder_name, tif_input_folder_name, compression_ratio, do_verify, do_delete_input_files, do_run_on_cluster) + % Converts each .mj2 file in input_folder_name to a multi-image .tif in + % output_folder_name. Will overwrite pre-existing files in + % output_folder_name, if present. + + if ~exist('compression_ratio', 'var') || isempty(compression_ratio) , + compression_ratio = 10 ; + end + if ~exist('do_verify', 'var') || isempty(do_verify) , + do_verify = false ; + end + if ~exist('do_delete_input_files', 'var') || isempty(do_delete_input_files) , + do_delete_input_files = false ; + end + if ~exist('do_run_on_cluster', 'var') || isempty(do_run_on_cluster) , + do_run_on_cluster = false ; + end + + if ~exist(mj2_output_folder_name, 'dir') , + mkdir(mj2_output_folder_name) ; + end + tif_input_entity_names = setdiff(simple_dir(tif_input_folder_name), {'.' '..'}) ; + tif_input_entity_count = length(tif_input_entity_names) ; + for i = 1 : tif_input_entity_count , + tif_input_entity_name = tif_input_entity_names{i} ; + tif_input_entity_path = fullfile(tif_input_folder_name, tif_input_entity_name) ; + if exist(tif_input_entity_path, 'dir') , + % if a folder, recurse + mj2_output_entity_path = fullfile(mj2_output_folder_name, tif_input_entity_name) ; + mj2_from_tif(mj2_output_entity_path, tif_input_entity_path, compression_ratio, do_verify, do_delete_input_files, do_run_on_cluster) ; + else + % if a normal file, convert to .mj2 if it's a .tif, or just + % copy otherwise + [~,~,ext] = fileparts(tif_input_entity_path) ; + if isequal(ext, '.tif') , + mj2_output_entity_path = fullfile(mj2_output_folder_name, replace_extension(tif_input_entity_name, '.mj2')) ; + if ~exist(mj2_output_entity_path, 'file') , + fprintf('Converting %s...\n', tif_input_entity_path); + tic_id = tic() ; + scratch_folder_path = get_scratch_folder_path() ; + temporary_tif_input_file_path = [tempname(scratch_folder_path) '.tif'] ; + temporary_mj2_output_file_path = [tempname(scratch_folder_path) '.mj2'] ; % risky, but VideoWriter is dumb and always adds this anyway + if do_run_on_cluster , + command_line = ... + sprintf(['matlab -nojvm ' ... + '-r "mj2_output_entity_path = ''%s''; tif_input_entity_path = ''%s''; do_verify = %s; ' ... + 'compression_ratio = %s; do_delete_input_file = %s; ' ... + 'temporary_tif_input_file_path = ''%s''; temporary_mj2_output_file_path = ''%s''; ' ... + 'copyfile(tif_input_entity_path, temporary_tif_input_file_path); ' ... + 'mj2_from_tif_single(temporary_mj2_output_file_path, temporary_tif_input_file_path, compression_ratio, do_verify); ' ... + 'copyfile(temporary_mj2_output_file_path, mj2_output_entity_path); ' ... + 'delete(temporary_tif_input_file_path); ' ... + 'delete(temporary_mj2_output_file_path); ' ... + 'if do_delete_input_file, delete(tif_input_entity_path); end; ' ... + 'exit"'], ... + mj2_output_entity_path, ... + tif_input_entity_path, ... + fif(do_verify, 'true', 'false'), ... + num2str(compression_ratio, '%.18g') , ... + fif(do_delete_input_files, 'true', 'false'), ... + temporary_tif_input_file_path, ... + temporary_mj2_output_file_path) ; + fprintf('Command line: %s\n', command_line) ; + %bsub_command_line = sprintf('bsub -P mouselight -o /dev/null -e /dev/null %s', command_line) ; + if do_verify , + core_count_to_request = 4 ; + else + core_count_to_request = 1 ; + end + bsub_command_line = sprintf('bsub -n%d -P mouselight %s', core_count_to_request, command_line) ; + fprintf('bsub Command line: %s\n', bsub_command_line) ; + system(bsub_command_line) ; + else + mj2_from_tif_single(mj2_output_entity_path, tif_input_entity_path, compression_ratio, do_verify, do_delete_input_files) ; + end + duration = toc(tic_id) ; + fprintf('Took %g seconds\n', duration) ; + end + else + mj2_output_entity_path = fullfile(mj2_output_folder_name, tif_input_entity_name) ; % input not really a tif, output not really a .mj2 + if ~exist(mj2_output_entity_path, 'file') , + copyfile(tif_input_entity_path, mj2_output_entity_path) ; + end + % Delete input file, if desired + if do_delete_input_files , + delete(tif_input_entity_path) ; + end + end + end + end + + % Delete the input folder if we were asked to delete input files, and if the input folder is really empty + if do_delete_input_files , + tif_input_entity_names = setdiff(simple_dir(tif_input_folder_name), {'.' '..'}) ; + tif_input_entity_count = length(tif_input_entity_names) ; + if tif_input_entity_count==0 , + rmdir(tif_input_folder_name) ; + end + end +end diff --git a/mj2_from_tif_single.m b/mj2_from_tif_single.m new file mode 100644 index 0000000..400a0db --- /dev/null +++ b/mj2_from_tif_single.m @@ -0,0 +1,37 @@ +function mj2_from_tif_single(mj2_output_file_name, tif_input_file_name, compression_ratio, do_verify, do_delete_input_file) + % Converts a single .tif file at input_file_name to a MJPEG-2000 file at + % output_file_name, using the given compression ratio. Will overwrite + % pre-existing file at output_file_name, if present. + + mj2_output_file_name + tif_input_file_name + + % Process input args + if ~exist('compression_ratio', 'var') || isempty(compression_ratio) , + compression_ratio = 10 ; + end + if ~exist('do_verify', 'var') || isempty(do_verify) , + do_verify = false ; + end + if ~exist('do_delete_input_file', 'var') || isempty(do_delete_input_file) , + do_delete_input_file = false ; + end + + % Read the input file + stack = read_16bit_grayscale_tif(tif_input_file_name) ; + + % Write the output file + write_16bit_grayscale_mj2(mj2_output_file_name, stack, compression_ratio) ; + + % Verify, if desired + if do_verify , + if ~is_mj2_similar_to_tif(mj2_output_file_name, tif_input_file_name) , + error('%s is not sufficiently similar to %s\n', mj2_output_file_name, tif_input_file_name) ; + end + end + + % Delete input file, if desired + if do_delete_input_file , + delete(tif_input_file_name) ; + end +end diff --git a/mj2tif_single.m b/mj2tif_single.m deleted file mode 100644 index 34abb72..0000000 --- a/mj2tif_single.m +++ /dev/null @@ -1,27 +0,0 @@ -function mj2tif_single(input_file_name, output_file_name, do_verify) - % Converts a single .mj2 file at input_file_name to a multi-image .tif - % at output_file_name. Will overwrite pre-existing file at - % output_file_name, if present. - - if nargin<3 || isempty(do_verify) , - do_verify = false ; - end - - input_file = VideoReader(input_file_name) ; - frame_index = 0 ; - while input_file.hasFrame() , - frame_index = frame_index + 1 ; - frame = input_file.readFrame() ; - if frame_index == 1 , - imwrite(frame, output_file_name, 'tif', 'WriteMode', 'overwrite', 'Compression', 'deflate') ; - else - imwrite(frame, output_file_name, 'tif', 'WriteMode', 'append', 'Compression', 'deflate') ; - end - end - - if do_verify , - if ~is_mj2_same_as_tif(input_file_name, output_file_name) , - error('%s is not the same as %s\n', output_file_name, input_file_name) ; - end - end -end diff --git a/modpath.m b/modpath.m new file mode 100644 index 0000000..24ebeb6 --- /dev/null +++ b/modpath.m @@ -0,0 +1,2 @@ +function modpath() +end diff --git a/test_freezing.m b/test_freezing.m new file mode 100644 index 0000000..dc0b973 --- /dev/null +++ b/test_freezing.m @@ -0,0 +1,9 @@ +path_to_this_script = mfilename('fullpath') ; +path_to_this_folder = fileparts(path_to_this_script) ; +mj2_output_folder_path = fullfile(path_to_this_folder, 'mj2-from-tif-test-output-folder') ; +tif_input_folder_path = fullfile(path_to_this_folder, 'mj2-from-tif-test-input-folder') ; +compression_ratio = 10 ; +do_verify = true ; +do_delete_input_files = true ; +do_run_on_cluster = true ; +mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, compression_ratio, do_verify, do_delete_input_files, do_run_on_cluster) diff --git a/thaw_2017_10_31_sample.m b/thaw_2017_10_31_sample.m index 05b589f..c7cb86d 100644 --- a/thaw_2017_10_31_sample.m +++ b/thaw_2017_10_31_sample.m @@ -1,3 +1,3 @@ do_verify = false ; do_on_cluster = true ; -mj2tif('/nrs/mouselight/SAMPLES/compressed_recovered/2017-10-31', '/groups/mousebrainmicro/mousebrainmicro/data/2017-10-31-thawed', do_verify, do_on_cluster) ; +tif_from_mj2('/groups/mousebrainmicro/mousebrainmicro/data/2017-10-31-thawed', '/nrs/mouselight/SAMPLES/compressed_recovered/2017-10-31', do_verify, do_on_cluster) ; diff --git a/thaw_2017_12_19_sample.m b/thaw_2017_12_19_sample.m index 685593d..bf2d831 100644 --- a/thaw_2017_12_19_sample.m +++ b/thaw_2017_12_19_sample.m @@ -1,3 +1,3 @@ do_verify = false ; do_on_cluster = true ; -mj2tif('/groups/mousebrainmicro/mousebrainmicro/data/2017-12-19', '/groups/mousebrainmicro/mousebrainmicro/data/2017-12-19-thawed', do_verify, do_on_cluster) ; +tif_from_mj2('/groups/mousebrainmicro/mousebrainmicro/data/2017-12-19-thawed', '/groups/mousebrainmicro/mousebrainmicro/data/2017-12-19', do_verify, do_on_cluster) ; diff --git a/mj2tif.m b/tif_from_mj2.m similarity index 54% rename from mj2tif.m rename to tif_from_mj2.m index d31dd5b..236d30f 100644 --- a/mj2tif.m +++ b/tif_from_mj2.m @@ -1,4 +1,4 @@ -function mj2tif(input_folder_name, output_folder_name, do_verify, do_run_on_cluster) +function tif_from_mj2(tif_output_folder_name, mj2_input_folder_name, do_verify, do_run_on_cluster) % Converts each .mj2 file in input_folder_name to a multi-image .tif in % output_folder_name. Will overwrite pre-existing files in % output_folder_name, if present. @@ -10,43 +10,43 @@ function mj2tif(input_folder_name, output_folder_name, do_verify, do_run_on_clus do_run_on_cluster = false ; end - if ~exist(output_folder_name, 'dir') , - mkdir(output_folder_name) ; + if ~exist(tif_output_folder_name, 'dir') , + mkdir(tif_output_folder_name) ; end - entity_names = setdiff(simple_dir(input_folder_name), {'.' '..'}) ; + entity_names = setdiff(simple_dir(mj2_input_folder_name), {'.' '..'}) ; entity_count = length(entity_names) ; for i = 1 : entity_count , - entity_name = entity_names{i} ; - entity_path = fullfile(input_folder_name, entity_name) ; - if exist(entity_path, 'dir') , + mj2_input_entity_name = entity_names{i} ; + mj2_input_entity_path = fullfile(mj2_input_folder_name, mj2_input_entity_name) ; + if exist(mj2_input_entity_path, 'dir') , % if a folder, recurse - output_entity_path = fullfile(output_folder_name, entity_name) ; - mj2tif(entity_path, output_entity_path, do_verify, do_run_on_cluster) ; + tif_output_entity_path = fullfile(tif_output_folder_name, mj2_input_entity_name) ; + tif_from_mj2(tif_output_entity_path, mj2_input_entity_path, do_verify, do_run_on_cluster) ; else % if a normal file, convert to .tif if it's a .mj2, or just % copy otherwise - [~,~,ext] = fileparts(entity_path) ; + [~,~,ext] = fileparts(mj2_input_entity_path) ; if isequal(ext, '.mj2') , - output_entity_path = fullfile(output_folder_name, replace_extension(entity_name, '.tif')) ; - if ~exist(output_entity_path, 'file') , - fprintf('Converting %s...\n', entity_path); + tif_output_entity_path = fullfile(tif_output_folder_name, replace_extension(mj2_input_entity_name, '.tif')) ; + if ~exist(tif_output_entity_path, 'file') , + fprintf('Converting %s...\n', mj2_input_entity_path); tic_id = tic() ; scratch_folder_path = get_scratch_folder_path() ; - temporary_input_file_path = tempname(scratch_folder_path) - temporary_output_file_path = tempname(scratch_folder_path) + temporary_input_file_path = tempname(scratch_folder_path) ; + temporary_output_file_path = tempname(scratch_folder_path) ; if do_run_on_cluster , command_line = ... sprintf(['matlab -nojvm -singleCompThread ' ... - '-r "entity_path = ''%s''; output_entity_path = ''%s''; do_verify = %s; ' ... + '-r "mj2_entity_path = ''%s''; tif_output_entity_path = ''%s''; do_verify = %s; ' ... 'temporary_input_file_path = ''%s''; temporary_output_file_path = ''%s''; ' ... - 'copyfile(entity_path, temporary_input_file_path); ' ... - 'mj2tif_single(temporary_input_file_path, temporary_output_file_path, do_verify); ' ... - 'copyfile(temporary_output_file_path, output_entity_path); ' ... + 'copyfile(mj2_entity_path, temporary_input_file_path); ' ... + 'tif_from_mj2_single(temporary_output_file_path, temporary_input_file_path, do_verify); ' ... + 'copyfile(temporary_output_file_path, tif_output_entity_path); ' ... 'delete(temporary_input_file_path); ' ... 'delete(temporary_output_file_path); ' ... 'exit"'], ... - entity_path, ... - output_entity_path, ... + mj2_input_entity_path, ... + tif_output_entity_path, ... fif(do_verify, 'true', 'false'), ... temporary_input_file_path, ... temporary_output_file_path) ; @@ -55,15 +55,15 @@ function mj2tif(input_folder_name, output_folder_name, do_verify, do_run_on_clus fprintf('bsub Command line: %s\n', bsub_command_line) ; system(bsub_command_line) ; else - mj2tif_single(entity_path, output_entity_path, do_verify) ; + tif_from_mj2_single(tif_output_entity_path, mj2_input_entity_path, do_verify) ; end duration = toc(tic_id) ; fprintf('Took %g seconds\n', duration) ; end else - output_entity_path = fullfile(output_folder_name, entity_name) ; - if ~exist(output_entity_path, 'file') , - copyfile(entity_path, output_entity_path) ; + tif_output_entity_path = fullfile(tif_output_folder_name, mj2_input_entity_name) ; + if ~exist(tif_output_entity_path, 'file') , + copyfile(mj2_input_entity_path, tif_output_entity_path) ; end end end diff --git a/tif_from_mj2_single.m b/tif_from_mj2_single.m new file mode 100644 index 0000000..6e11920 --- /dev/null +++ b/tif_from_mj2_single.m @@ -0,0 +1,27 @@ +function tif_from_mj2_single(tif_output_file_name, mj2_input_file_name, do_verify) + % Converts a single .mj2 file at input_file_name to a multi-image .tif + % at output_file_name. Will overwrite pre-existing file at + % output_file_name, if present. + + if ~exist('do_verify', 'var') || isempty(do_verify) , + do_verify = false ; + end + + input_file = VideoReader(mj2_input_file_name) ; + frame_index = 0 ; + while input_file.hasFrame() , + frame_index = frame_index + 1 ; + frame = input_file.readFrame() ; + if frame_index == 1 , + imwrite(frame, tif_output_file_name, 'tif', 'WriteMode', 'overwrite', 'Compression', 'deflate') ; + else + imwrite(frame, tif_output_file_name, 'tif', 'WriteMode', 'append', 'Compression', 'deflate') ; + end + end + + if do_verify , + if ~is_mj2_same_as_tif(mj2_input_file_name, tif_output_file_name) , + error('%s is not the same as %s\n', tif_output_file_name, mj2_input_file_name) ; + end + end +end diff --git a/write_16bit_grayscale_mj2.m b/write_16bit_grayscale_mj2.m new file mode 100644 index 0000000..d981aa1 --- /dev/null +++ b/write_16bit_grayscale_mj2.m @@ -0,0 +1,12 @@ +function write_16bit_grayscale_mj2(file_name, stack, compression_ratio) + fprintf('Nominal file name in write_16bit_grayscale_mj2(): %s\n', file_name) ; + [row_count, column_count, page_count] = size(stack) ; + stack_for_video_writer = reshape(stack, [row_count, column_count 1 page_count]) ; + % VideoWriter supports RGB, so have time in 4th dimension to disambiguate. + vw = VideoWriter(file_name, 'Motion JPEG 2000') ; + vw.CompressionRatio = compression_ratio ; + vw.MJ2BitDepth = 16 ; + vw.open() ; + vw.writeVideo(stack_for_video_writer) ; + vw.close() ; +end diff --git a/write_16bit_grayscalr_mj2.m b/write_16bit_grayscalr_mj2.m new file mode 100644 index 0000000..14501e6 --- /dev/null +++ b/write_16bit_grayscalr_mj2.m @@ -0,0 +1,8 @@ +function write_16bit_grayscalr_mj2(file_name, stack) + vw = VideoWriter(file_name, 'Motion JPEG 2000') ; + vw.CompressionRatio = compression_ratio ; + vw.MJ2BitDepth = 16 ; + vw.open() ; + vw.writeVideo(stack) ; + vw.close() ; +end