diff --git a/batch_preverify_tif_files_after_mj2_from_tif.m b/batch_preverify_tif_files_after_mj2_from_tif.m index 6677370..fb03da2 100644 --- a/batch_preverify_tif_files_after_mj2_from_tif.m +++ b/batch_preverify_tif_files_after_mj2_from_tif.m @@ -4,7 +4,7 @@ function batch_preverify_tif_files_after_mj2_from_tif(mj2_folder_name, tif_folde bsub_options = '-n4 -P mouselight -W 59 -J verify-frozen' ; find_and_batch(tif_folder_name, ... - @does_file_name_end_in_dot_tif, ... + @does_need_preverification_after_freezing, ... @preverify_single_tif_file_after_mj2_from_tif, ... do_submit, ... bsub_options, ... diff --git a/does_is_similar_to_tif_check_file_exist.m b/does_is_similar_to_tif_check_file_exist.m new file mode 100644 index 0000000..4af3ef8 --- /dev/null +++ b/does_is_similar_to_tif_check_file_exist.m @@ -0,0 +1,7 @@ +function result = does_is_similar_to_tif_check_file_exist(tif_file_path, tif_root_path, mj2_root_path) + tif_relative_path = relpath(tif_file_path, tif_root_path) ; + mj2_relative_path = replace_extension(tif_relative_path, '.mj2') ; + mj2_file_path = fullfile(mj2_root_path, mj2_relative_path) ; + check_file_path = horzcat(mj2_file_path, '.is-similar-to-tif') ; + result = logical(exist(check_file_path, 'file')) ; +end diff --git a/does_need_preverification_after_freezing.m b/does_need_preverification_after_freezing.m new file mode 100644 index 0000000..cb99f5d --- /dev/null +++ b/does_need_preverification_after_freezing.m @@ -0,0 +1,5 @@ +function result = does_need_preverification_after_freezing(tif_file_path, tif_root_path, mj2_root_path) + [~,file_name] = fileparts2(tif_file_path) ; + result = does_file_name_end_in_dot_tif(file_name) && ... + ~does_is_similar_to_tif_check_file_exist(tif_file_path, tif_root_path, mj2_root_path) ; +end diff --git a/freeze_2018_10_01.m b/freeze_2018_10_01.m new file mode 100644 index 0000000..3476095 --- /dev/null +++ b/freeze_2018_10_01.m @@ -0,0 +1,9 @@ +sample_date = '2018-10-01' ; +mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; +tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; +compression_ratio = 10 ; +do_verify = false ; +do_run_on_cluster = true ; + +mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, compression_ratio, do_verify, do_run_on_cluster) +is_all_well = verify_frozen_files(mj2_output_folder_path, tif_input_folder_path, do_run_on_cluster) diff --git a/freeze_2018_12_01.m b/freeze_2018_12_01.m new file mode 100644 index 0000000..c153672 --- /dev/null +++ b/freeze_2018_12_01.m @@ -0,0 +1,9 @@ +sample_date = '2018-12-01' ; +mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s/Tiling', sample_date) ; +tif_input_folder_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_1_line_fix_output', sample_date) ; +compression_ratio = 10 ; +do_verify = false ; +do_run_on_cluster = true ; + +mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, compression_ratio, do_verify, do_run_on_cluster) +is_all_well = verify_frozen_files(mj2_output_folder_path, tif_input_folder_path, do_run_on_cluster) diff --git a/freeze_2018_12_01_sample.m b/freeze_2018_12_01_sample.m deleted file mode 100644 index c7cb86d..0000000 --- a/freeze_2018_12_01_sample.m +++ /dev/null @@ -1,3 +0,0 @@ -do_verify = false ; -do_on_cluster = true ; -tif_from_mj2('/groups/mousebrainmicro/mousebrainmicro/data/2017-10-31-thawed', '/nrs/mouselight/SAMPLES/compressed_recovered/2017-10-31', do_verify, do_on_cluster) ; diff --git a/freeze_sample.m b/freeze_sample.m new file mode 100644 index 0000000..68bf87f --- /dev/null +++ b/freeze_sample.m @@ -0,0 +1,42 @@ +mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, compression_ratio, do_verify, do_run_on_cluster) +%is_all_well = verify_frozen_files(mj2_output_folder_path, tif_input_folder_path, do_run_on_cluster) + +fprintf('Doing batch preverification of .tif files...\n') ; +batch_preverify_tif_files_after_mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, do_run_on_cluster) +fprintf('Done with batch preverification of .tif files.\n') ; + +fprintf('Doing verification of .tif files...\n') ; +verify_tif_files_after_mj2_from_tif(tif_input_folder_path, mj2_output_folder_path) ; +fprintf('Done with verification of .tif files.\n') ; + +fprintf('Doing verification of non-.tif files...\n') ; +verify_non_tif_files_after_mj2_from_tif(mj2_output_folder_path, tif_input_folder_path) ; +fprintf('Done with verification of non-.tif files.\n') ; + +% Delete certificate files (just run locally) +find_and_batch(mj2_output_folder_path, ... + @does_file_name_match_certificate_file, ... + @delete, ... + false, ... + '') ; + +% Do a final comparison of .tif files and .mj2 files, just for the heck of it +[status, result] = system(sprintf('find %s -name "*.tif" -print | wc -l', tif_input_folder_path)) ; +if status~= 0 , + error('Unable to count number of .tif files\n') ; +end +final_tif_count = str2double(result) ; +fprintf('.tif count in %s is: %d\n', tif_input_folder_path, final_tif_count) ; + +[status, result] = system(sprintf('find %s -name "*.mj2" -print | wc -l', mj2_output_folder_path)) ; +if status~= 0 , + error('Unable to count final number of .mj2 files\n') ; +end +final_mj2_count = str2double(result) ; +fprintf('Final .mj2 count in %s is: %d\n', mj2_output_folder_path, final_mj2_count) ; + +% Report final status if we get this far +fprintf('Compression and verification succeeded!\n') ; + + + diff --git a/mj2_from_tif.m b/mj2_from_tif.m index 4ea971a..d50f252 100644 --- a/mj2_from_tif.m +++ b/mj2_from_tif.m @@ -18,7 +18,7 @@ function mj2_from_tif(mj2_output_folder_name, tif_input_folder_name, compression job_ids = mj2_from_tif_helper(mj2_output_folder_name, tif_input_folder_name, compression_ratio, do_verify, do_run_on_cluster, zeros(1,0)) ; % Wait for the jobs to finish - fprintf('Waiting for mj2_from_tif() bjobs to finish...\n') ; + fprintf('Waiting for %d mj2_from_tif() bjobs to finish...\n', length(job_ids)) ; bwait(job_ids) ; fprintf('mj2_from_tif() bjobs are done.\n') ; @@ -32,7 +32,7 @@ function mj2_from_tif(mj2_output_folder_name, tif_input_folder_name, compression if ~exist(mj2_output_folder_name, 'dir') , mkdir(mj2_output_folder_name) ; end - tif_input_entity_names = setdiff(simple_dir(tif_input_folder_name), {'.' '..'}) ; + tif_input_entity_names = simple_dir(tif_input_folder_name) ; tif_input_entity_count = length(tif_input_entity_names) ; for i = 1 : tif_input_entity_count , tif_input_entity_name = tif_input_entity_names{i} ; diff --git a/mouselight_toolbox b/mouselight_toolbox index f3e21cb..1eed87f 160000 --- a/mouselight_toolbox +++ b/mouselight_toolbox @@ -1 +1 @@ -Subproject commit f3e21cb8cd085f78f89e3e2aa47ff81489b46c49 +Subproject commit 1eed87feb01430dcdae119f19ec9fa5b03066867 diff --git a/preverify_single_tif_file_after_mj2_from_tif.m b/preverify_single_tif_file_after_mj2_from_tif.m index 879cee7..46ba28f 100644 --- a/preverify_single_tif_file_after_mj2_from_tif.m +++ b/preverify_single_tif_file_after_mj2_from_tif.m @@ -17,6 +17,10 @@ function preverify_single_tif_file_after_mj2_from_tif(tif_file_path, tif_root_fo tif_stack = read_16bit_grayscale_tif(tif_file_path) ; if is_mj2_similar_to_tif(mj2_stack, tif_stack) , touch(check_file_path) ; + else + %nop() ; end + else + %nop() ; end end diff --git a/test_freezing.m b/test_freezing.m deleted file mode 100644 index c9a792c..0000000 --- a/test_freezing.m +++ /dev/null @@ -1,76 +0,0 @@ -path_to_this_script = mfilename('fullpath') ; -path_to_this_folder = fileparts(path_to_this_script) ; -mj2_output_folder_path = fullfile(path_to_this_folder, 'freezing-test-output-folder') ; -tif_input_folder_path = fullfile(path_to_this_folder, 'freezing-test-input-folder') ; -compression_ratio = 10 ; -do_verify = false ; -do_run_on_cluster = true ; - -% Delete the test output folder -reset_for_test(mj2_output_folder_path) ; - -% Copy the source folder to the destination, compressing and .tif's to .mj2's -mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, compression_ratio, do_verify, do_run_on_cluster) -%is_all_well = verify_frozen_files(mj2_output_folder_path, tif_input_folder_path, do_run_on_cluster) - -% For each .tif, check that a close-enough .mj2 exists, and write a certificate -% file if it does -fprintf('Doing batch preverification of .tif files...\n') ; -batch_preverify_tif_files_after_mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, do_run_on_cluster) -fprintf('Done with batch preverification of .tif files.\n') ; - -% Check that each .tif has a corresponding certificate file -fprintf('Doing verification of .tif files...\n') ; -is_all_well = verify_tif_files_after_mj2_from_tif(tif_input_folder_path, mj2_output_folder_path) ; -fprintf('Done with verification of .tif files.\n') ; - -% If anything is amiss, exit -if ~is_all_well , - fprintf('Compression and/or verification failed.\n') ; - return -end - -% Check the non-tif files (must be exactly same is destination) -fprintf('Doing verification of non-.tif files...\n') ; -is_all_well = verify_non_tif_files_after_mj2_from_tif(mj2_output_folder_path, tif_input_folder_path) ; -fprintf('Done with verification of non-.tif files.\n') ; - -% If anything is amiss, exit -if ~is_all_well , - fprintf('Compression and/or verification failed.\n') ; - return -end - -% Delete certificate files -find_and_batch(mj2_output_folder_path, ... - @does_file_name_match_certificate_file, ... - @delete, ... - false, ... - '') ; - -% Do a final comparison of .tif files and .mj2 files, just for the heck of it -[status, result] = system(sprintf('find %s -name "*.tif" -print | wc -l', tif_input_folder_path)) ; -if status~= 0 , - is_all_well = false ; - fprintf('Unable to count number of .tif files\n') ; - return -end -final_tif_count = str2double(result) ; -fprintf('.tif count in %s is: %d\n', tif_input_folder_path, final_tif_count) ; - -[status, result] = system(sprintf('find %s -name "*.mj2" -print | wc -l', mj2_output_folder_path)) ; -if status~= 0 , - is_all_well = false ; - fprintf('Unable to count final number of .mj2 files\n') ; - return -end -final_mj2_count = str2double(result) ; -fprintf('Final .mj2 count in %s is: %d\n', mj2_output_folder_path, final_mj2_count) ; - -% Report final status -if is_all_well , - fprintf('Compression and verification succeeded!\n') ; -else - fprintf('Compression and/or verification failed.\n') ; -end - diff --git a/test_freezing_locally.m b/test_freezing_locally.m new file mode 100644 index 0000000..dc86ad0 --- /dev/null +++ b/test_freezing_locally.m @@ -0,0 +1,13 @@ +path_to_this_script = mfilename('fullpath') ; +path_to_this_folder = fileparts(path_to_this_script) ; +mj2_output_folder_path = fullfile(path_to_this_folder, 'freezing-test-output-folder') ; +tif_input_folder_path = fullfile(path_to_this_folder, 'freezing-test-input-folder') ; +compression_ratio = 10 ; +do_verify = false ; +do_run_on_cluster = false ; + +% Delete the test output folder +reset_for_test(mj2_output_folder_path) ; + +% Call the script +freeze_sample diff --git a/test_freezing_on_cluster.m b/test_freezing_on_cluster.m new file mode 100644 index 0000000..b3972db --- /dev/null +++ b/test_freezing_on_cluster.m @@ -0,0 +1,13 @@ +path_to_this_script = mfilename('fullpath') ; +path_to_this_folder = fileparts(path_to_this_script) ; +mj2_output_folder_path = fullfile(path_to_this_folder, 'freezing-test-output-folder') ; +tif_input_folder_path = fullfile(path_to_this_folder, 'freezing-test-input-folder') ; +compression_ratio = 10 ; +do_verify = false ; +do_run_on_cluster = true ; + +% Delete the test output folder +reset_for_test(mj2_output_folder_path) ; + +% Call the script +freeze_sample diff --git a/verify_non_tif_files_after_mj2_from_tif.m b/verify_non_tif_files_after_mj2_from_tif.m index be91ccc..da8b2f9 100644 --- a/verify_non_tif_files_after_mj2_from_tif.m +++ b/verify_non_tif_files_after_mj2_from_tif.m @@ -1,11 +1,10 @@ -function is_all_well = verify_non_tif_files_after_mj2_from_tif(mj2_root_folder_path, tif_root_folder_path) +function verify_non_tif_files_after_mj2_from_tif(mj2_root_folder_path, tif_root_folder_path) % Makes sure all the target files are present in the the destination, and have % the right size, or at least a plausible size in the case of .mj2's. - is_all_well = ... - find_and_verify(tif_root_folder_path, ... - @does_file_name_not_end_in_dot_tif, ... - @verify_single_non_tif_file_after_mj2_from_tif, ... - tif_root_folder_path, ... - mj2_root_folder_path) ; + find_and_verify(tif_root_folder_path, ... + @does_file_name_not_end_in_dot_tif, ... + @verify_single_non_tif_file_after_mj2_from_tif, ... + tif_root_folder_path, ... + mj2_root_folder_path) ; end diff --git a/verify_tif_files_after_mj2_from_tif.m b/verify_tif_files_after_mj2_from_tif.m index 06d6a77..cb34b96 100644 --- a/verify_tif_files_after_mj2_from_tif.m +++ b/verify_tif_files_after_mj2_from_tif.m @@ -1,10 +1,10 @@ -function is_all_well = verify_tif_files_after_mj2_from_tif(tif_root_folder_path, mj2_root_folder_path) +function verify_tif_files_after_mj2_from_tif(tif_root_folder_path, mj2_root_folder_path) % Makes sure there's a .similar-mj2-exists for each .tif file in % tif_root_folder_path. - is_all_well = find_and_verify(tif_root_folder_path, ... - @does_file_name_end_in_dot_tif, ... - @verify_single_tif_file_after_mj2_from_tif, ... - tif_root_folder_path, ... - mj2_root_folder_path) ; + find_and_verify(tif_root_folder_path, ... + @does_file_name_end_in_dot_tif, ... + @verify_single_tif_file_after_mj2_from_tif, ... + tif_root_folder_path, ... + mj2_root_folder_path) ; end \ No newline at end of file