From 743a20a08ff0c41bed2803786007b4f41745a20f Mon Sep 17 00:00:00 2001 From: "Adam L. Taylor" Date: Fri, 19 May 2023 20:45:13 -0400 Subject: [PATCH] master: Changes to exclude ktx/ folder --- batch_convert_leaf_mj2_files_to_tif.m | 22 --- batch_convert_leaf_tif_files_to_mj2.m | 23 --- ...tch_preverify_stack_files_after_freezing.m | 12 +- ... => batch_verify_tif_files_after_thawing.m | 16 +- check_certs_after_verifying_thawing.m | 12 +- check_single_cert_after_verifying_thawing.m | 8 +- copy_non_stack_files_during_freezing.m | 30 ++++ ...m => copy_non_stack_files_during_thawing.m | 21 ++- copy_non_tif_files_to_mj2_side.m | 22 --- does_is_similar_to_mj2_check_file_exist.m | 7 + does_need_preverification_after_freezing.m | 2 +- find_and_count.m | 99 ++++++++----- freeze_mouselight_folder.m | 132 ++++++++++------- ...ut.m => is_mj2_and_lacks_cert_for_output.m | 8 +- is_not_the_ktx_folder.m | 16 ++ mouselight_toolbox | 2 +- thaw_mouselight_folder.m | 137 ++++++++++-------- verify_non_stack_files_after_freezing.m | 13 ++ verify_non_stack_files_after_thawing.m | 13 ++ verify_non_tif_files_after_mj2_from_tif.m | 10 -- verify_non_tif_files_after_thawing.m | 10 -- verify_single_non_stack_file.m | 8 +- ...y_single_non_tif_file_after_mj2_from_tif.m | 10 -- verify_single_tif_file_after_mj2_from_tif.m | 7 +- verify_tif_files_after_mj2_from_tif.m | 10 +- 25 files changed, 357 insertions(+), 293 deletions(-) delete mode 100644 batch_convert_leaf_mj2_files_to_tif.m delete mode 100644 batch_convert_leaf_tif_files_to_mj2.m rename batch_preverify_tif_files_after_mj2_from_tif.m => batch_preverify_stack_files_after_freezing.m (66%) rename batch_verify_leaf_tif_files_after_thawing.m => batch_verify_tif_files_after_thawing.m (58%) create mode 100644 copy_non_stack_files_during_freezing.m rename copy_non_mj2_files_to_tif_side.m => copy_non_stack_files_during_thawing.m (51%) delete mode 100644 copy_non_tif_files_to_mj2_side.m create mode 100644 does_is_similar_to_mj2_check_file_exist.m rename is_leaf_mj2_and_lacks_cert_for_output.m => is_mj2_and_lacks_cert_for_output.m (65%) create mode 100644 is_not_the_ktx_folder.m create mode 100644 verify_non_stack_files_after_freezing.m create mode 100644 verify_non_stack_files_after_thawing.m delete mode 100644 verify_non_tif_files_after_mj2_from_tif.m delete mode 100644 verify_non_tif_files_after_thawing.m delete mode 100644 verify_single_non_tif_file_after_mj2_from_tif.m diff --git a/batch_convert_leaf_mj2_files_to_tif.m b/batch_convert_leaf_mj2_files_to_tif.m deleted file mode 100644 index 097bbeb..0000000 --- a/batch_convert_leaf_mj2_files_to_tif.m +++ /dev/null @@ -1,22 +0,0 @@ -function batch_convert_leaf_mj2_files_to_tif(output_folder_name, ... - input_folder_name, ... - do_submit, ... - do_try, ... - maximum_running_slot_count, ... - submit_host_name) - % Find all .mj2 files in folders with no subfolders, convert them to .tif - - slots_per_job = 1 ; - bsub_options = '-P mouselight -W 59 -J thaw' ; - find_and_batch(input_folder_name, ... - @is_a_mj2_in_a_folder_with_no_subfolders_and_target_missing, ... - @tif_from_mj2_single_for_find_and_batch, ... - 'do_submit', do_submit, ... - 'do_try', do_try, ... - 'submit_host_name', submit_host_name, ... - 'maximum_running_slot_count', maximum_running_slot_count, ... - 'slots_per_job', slots_per_job, ... - 'bsub_options', bsub_options, ... - 'predicate_extra_args', {input_folder_name, output_folder_name}, ... - 'batch_function_extra_args', {input_folder_name, output_folder_name}) ; -end diff --git a/batch_convert_leaf_tif_files_to_mj2.m b/batch_convert_leaf_tif_files_to_mj2.m deleted file mode 100644 index 647df7c..0000000 --- a/batch_convert_leaf_tif_files_to_mj2.m +++ /dev/null @@ -1,23 +0,0 @@ -function batch_convert_leaf_tif_files_to_mj2(mj2_folder_name, ... - tif_folder_name, ... - compression_ratio, ... - do_submit, ... - do_try, ... - maximum_running_slot_count, ... - submit_host_name) - % Find all .tif files in folders with no subfolders, convert them to .mj2 - - slots_per_job = 1 ; - bsub_options = '-P mouselight -W 59 -J freeze' ; - find_and_batch(tif_folder_name, ... - @is_a_tif_in_a_folder_with_no_subfolders_and_target_missing, ... - @mj2_from_tif_single_for_find_and_batch, ... - 'do_submit', do_submit, ... - 'do_try', do_try, ... - 'submit_host_name', submit_host_name, ... - 'maximum_running_slot_count', maximum_running_slot_count, ... - 'slots_per_job', slots_per_job, ... - 'bsub_options', bsub_options, ... - 'predicate_extra_args', {tif_folder_name, mj2_folder_name}, ... - 'batch_function_extra_args', {tif_folder_name, mj2_folder_name, compression_ratio}) ; -end diff --git a/batch_preverify_tif_files_after_mj2_from_tif.m b/batch_preverify_stack_files_after_freezing.m similarity index 66% rename from batch_preverify_tif_files_after_mj2_from_tif.m rename to batch_preverify_stack_files_after_freezing.m index b655d14..427966b 100644 --- a/batch_preverify_tif_files_after_mj2_from_tif.m +++ b/batch_preverify_stack_files_after_freezing.m @@ -1,9 +1,9 @@ -function batch_preverify_tif_files_after_mj2_from_tif(mj2_folder_name, ... - tif_folder_name, ... - do_submit, ... - do_try, ... - submit_host_name, ... - maximum_running_slot_count) +function batch_preverify_stack_files_after_freezing(mj2_folder_name, ... + tif_folder_name, ... + do_submit, ... + do_try, ... + submit_host_name, ... + maximum_running_slot_count) % Makes sure all the big target files are present in the the destination, and % are similar to their source files. diff --git a/batch_verify_leaf_tif_files_after_thawing.m b/batch_verify_tif_files_after_thawing.m similarity index 58% rename from batch_verify_leaf_tif_files_after_thawing.m rename to batch_verify_tif_files_after_thawing.m index 17e0ec8..2499a12 100644 --- a/batch_verify_leaf_tif_files_after_thawing.m +++ b/batch_verify_tif_files_after_thawing.m @@ -1,16 +1,16 @@ -function batch_verify_leaf_tif_files_after_thawing(output_folder_name, ... - input_folder_name, ... - do_submit, ... - do_try, ... - submit_host_name, ... - maximum_running_slot_count) +function batch_verify_tif_files_after_thawing(output_folder_name, ... + input_folder_name, ... + do_submit, ... + do_try, ... + submit_host_name, ... + maximum_running_slot_count) % Makes sure all the leaf image files are present in the the destination, and % are similar to their source files. slots_per_job = 4 ; - bsub_options = '-P mouselight -W 59 -J verify-thawed-leaves' ; + bsub_options = '-P mouselight -W 59 -J verify-thawed-stack' ; find_and_batch(input_folder_name, ... - @is_leaf_mj2_and_lacks_cert_for_output, ... + @is_mj2_and_lacks_cert_for_output, ... @verify_single_tif_file_after_thawing, ... 'do_submit', do_submit, ... 'do_try', do_try, ... diff --git a/check_certs_after_verifying_thawing.m b/check_certs_after_verifying_thawing.m index 40b5298..13bbc8b 100644 --- a/check_certs_after_verifying_thawing.m +++ b/check_certs_after_verifying_thawing.m @@ -1,10 +1,14 @@ function check_certs_after_verifying_thawing(input_root_folder_path, output_root_folder_path) % Makes sure there's a .similar-tif-exists for each .mj2 file in % input_root_folder_path. + + function result = check_single_cert_after_verifying_thawing_wrapper(root_folder_path, base_folder_relative_path, file_name, depth) %#ok + result = check_single_cert_after_verifying_thawing(root_folder_path, base_folder_relative_path, file_name, output_root_folder_path) ; + end + find_and_verify(... input_root_folder_path, ... - @is_a_mj2_in_a_folder_with_no_subfolders, ... - @check_single_cert_after_verifying_thawing, ... - input_root_folder_path, ... - output_root_folder_path) ; + @does_file_name_end_in_dot_mj2, ... + @check_single_cert_after_verifying_thawing_wrapper, ... + @(varargin)(true)) ; end diff --git a/check_single_cert_after_verifying_thawing.m b/check_single_cert_after_verifying_thawing.m index 5828ba9..bab83cc 100644 --- a/check_single_cert_after_verifying_thawing.m +++ b/check_single_cert_after_verifying_thawing.m @@ -1,7 +1,7 @@ -function result = check_single_cert_after_verifying_thawing(input_file_path, input_root_folder_path, output_root_folder_path) - input_relative_file_path = relpath(input_file_path, input_root_folder_path) ; - output_relative_file_path = replace_extension(input_relative_file_path, '.tif') ; - output_file_path = fullfile(output_root_folder_path, output_relative_file_path) ; +function result = check_single_cert_after_verifying_thawing(... + input_root_folder_path, input_folder_relative_path, input_file_name, output_root_folder_path) %#ok + output_file_name = replace_extension(input_file_name, '.tif') ; + output_file_path = fullfile(output_root_folder_path, input_folder_relative_path, output_file_name) ; cert_file_path = horzcat(output_file_path, '.is-similar-to-mj2') ; result = logical( exist(cert_file_path, 'file') ) ; end diff --git a/copy_non_stack_files_during_freezing.m b/copy_non_stack_files_during_freezing.m new file mode 100644 index 0000000..6139c0a --- /dev/null +++ b/copy_non_stack_files_during_freezing.m @@ -0,0 +1,30 @@ +function copy_non_stack_files_during_freezing(output_folder_name, ... + input_folder_name) + % Find all non-.tif files, copy them over to the mj2 side + + do_submit = false ; + do_try = true ; + maximum_running_slot_count = 400 ; % doesn't matter + slots_per_job = 1 ; % doesn't matter + bsub_options = '-P mouselight -W 59 -J copy' ; % doesn't matter + submit_host_name = '' ; + + function result = predicate_function(tif_side_file_path) + result = is_a_non_tif_and_target_missing(tif_side_file_path, input_folder_name, output_folder_name) ; + end + + function batch_function(tif_side_file_path) + copy_file_for_find_and_batch(tif_side_file_path, input_folder_name, output_folder_name) ; + end + + find_and_batch(input_folder_name, ... + @predicate_function, ... + @batch_function, ... + 'do_submit', do_submit, ... + 'do_try', do_try, ... + 'submit_host_name', submit_host_name, ... + 'maximum_running_slot_count', maximum_running_slot_count, ... + 'slots_per_job', slots_per_job, ... + 'bsub_options', bsub_options, ... + 'folder_predicate_function', @is_not_the_ktx_folder) ; +end diff --git a/copy_non_mj2_files_to_tif_side.m b/copy_non_stack_files_during_thawing.m similarity index 51% rename from copy_non_mj2_files_to_tif_side.m rename to copy_non_stack_files_during_thawing.m index f5dc20a..82a0b20 100644 --- a/copy_non_mj2_files_to_tif_side.m +++ b/copy_non_stack_files_during_thawing.m @@ -1,5 +1,5 @@ -function copy_non_mj2_files_to_tif_side(output_folder_name, ... - input_folder_name) +function copy_non_stack_files_during_thawing(output_folder_name, ... + input_folder_name) % Find all non-.mj2 files, copy them over to the tif side do_submit = false ; @@ -8,15 +8,22 @@ function copy_non_mj2_files_to_tif_side(output_folder_name, ... slots_per_job = 1 ; % doesn't matter bsub_options = '-P mouselight -W 59 -J copy' ; % doesn't matter submit_host_name = '' ; + + function result = predicate_function(mj2_side_file_path) + result = is_a_non_mj2_and_target_missing(mj2_side_file_path, input_folder_name, output_folder_name) ; + end + + function batch_function(mj2_side_file_path) + copy_file_for_find_and_batch(mj2_side_file_path, input_folder_name, output_folder_name) ; + end + find_and_batch(input_folder_name, ... - @is_a_non_mj2_and_target_missing, ... - @copy_file_for_find_and_batch, ... + @predicate_function, ... + @batch_function, ... 'do_submit', do_submit, ... 'do_try', do_try, ... 'submit_host_name', submit_host_name, ... 'maximum_running_slot_count', maximum_running_slot_count, ... 'slots_per_job', slots_per_job, ... - 'bsub_options', bsub_options, ... - 'predicate_extra_args', {input_folder_name, output_folder_name}, ... - 'batch_function_extra_args', {input_folder_name, output_folder_name}) ; + 'bsub_options', bsub_options) ; end diff --git a/copy_non_tif_files_to_mj2_side.m b/copy_non_tif_files_to_mj2_side.m deleted file mode 100644 index f7b3feb..0000000 --- a/copy_non_tif_files_to_mj2_side.m +++ /dev/null @@ -1,22 +0,0 @@ -function copy_non_tif_files_to_mj2_side(mj2_folder_name, ... - tif_folder_name) - % Find all non-.tif files, copy them over to the mj2 side - - do_submit = false ; - do_try = true ; - maximum_running_slot_count = 400 ; % doesn't matter - slots_per_job = 1 ; % doesn't matter - bsub_options = '-P mouselight -W 59 -J copy' ; % doesn't matter - submit_host_name = '' ; - find_and_batch(tif_folder_name, ... - @is_a_non_tif_and_target_missing, ... - @copy_file_for_find_and_batch, ... - 'do_submit', do_submit, ... - 'do_try', do_try, ... - 'submit_host_name', submit_host_name, ... - 'maximum_running_slot_count', maximum_running_slot_count, ... - 'slots_per_job', slots_per_job, ... - 'bsub_options', bsub_options, ... - 'predicate_extra_args', {tif_folder_name, mj2_folder_name} , ... - 'batch_function_extra_args', {tif_folder_name, mj2_folder_name} ) ; -end diff --git a/does_is_similar_to_mj2_check_file_exist.m b/does_is_similar_to_mj2_check_file_exist.m new file mode 100644 index 0000000..9782d0f --- /dev/null +++ b/does_is_similar_to_mj2_check_file_exist.m @@ -0,0 +1,7 @@ +function result = does_is_similar_to_mj2_check_file_exist(tif_file_path, mj2_root_path, tif_root_path) + mj2_relative_path = relpath(tif_file_path, mj2_root_path) ; + tif_relative_path = replace_extension(mj2_relative_path, '.tif') ; + tif_file_path = fullfile(tif_root_path, tif_relative_path) ; + check_file_path = horzcat(tif_file_path, '.is-similar-to-mj2') ; + result = logical(exist(check_file_path, 'file')) ; +end diff --git a/does_need_preverification_after_freezing.m b/does_need_preverification_after_freezing.m index f4dc117..574aee8 100644 --- a/does_need_preverification_after_freezing.m +++ b/does_need_preverification_after_freezing.m @@ -1,4 +1,4 @@ function result = does_need_preverification_after_freezing(tif_file_path, tif_root_path, mj2_root_path) - result = is_a_tif_and_target_present(tif_file_path, tif_root_path, mj2_root_path) && ... + result = does_file_name_end_in_dot_tif(tif_file_path) && ... ~does_is_similar_to_tif_check_file_exist(tif_file_path, tif_root_path, mj2_root_path) ; end diff --git a/find_and_count.m b/find_and_count.m index c803803..3787aaf 100644 --- a/find_and_count.m +++ b/find_and_count.m @@ -1,37 +1,70 @@ -function result = find_and_count(base_folder_path, is_file_a_keeper_predicate, varargin) - result = ... - find_and_list_helper(base_folder_path, ... - is_file_a_keeper_predicate, ... - 0, ... - varargin{:}) ; -end +function result = find_and_count(root_folder_path, is_file_a_keeper_predicate, folder_filter_function) + seed = 0 ; % for counting, start with a count of zero + + function result = filter(root_folder_path, base_folder_relative_path, file_name, depth, is_this_file_a_folder) %#ok + result = feval(is_file_a_keeper_predicate, root_folder_path, base_folder_relative_path, file_name) ; + end + result = file_accumulate_map_filter(root_folder_path, seed, @accumulator, @mapper, @filter, folder_filter_function) ; +end +function result = accumulator(result_so_far, file_value) + % For counting, just add the file_value to the ongoing count + result = result_so_far + file_value ; +end -function result = ... - find_and_list_helper(base_folder_path, ... - is_file_a_keeper_predicate, ... - incoming_count, ... - varargin) - count_so_far = incoming_count ; - [file_names, is_file_a_folder] = simple_dir(base_folder_path) ; - file_count = length(file_names) ; - for i = 1 : file_count , - file_name = file_names{i} ; - is_this_file_a_folder = is_file_a_folder(i) ; - file_path = fullfile(base_folder_path, file_name) ; - if is_this_file_a_folder , - % if a folder, recurse - count_so_far = ... - find_and_list_helper(file_path, ... - is_file_a_keeper_predicate, ... - count_so_far, ... - varargin{:}) ; - else - if feval(is_file_a_keeper_predicate, file_path, varargin{:}) , - count_so_far = count_so_far + 1 ; - end - end - end - result = count_so_far ; +function result = mapper(root_folder_path, base_folder_relative_path, file_name, depth) %#ok + % For counting, just want to return one for each file that gets past the + % filter + result = 1 ; end + + +% function result = find_and_count(base_folder_path, is_file_a_keeper_predicate, varargin) +% [folder_predicate_function] = ... +% parse_keyword_args(... +% varargin, ... +% 'folder_predicate_function', @(folder_path, depth)(true)) ; +% +% result = ... +% find_and_count_helper(base_folder_path, ... +% is_file_a_keeper_predicate, ... +% folder_predicate_function, ... +% 0, ... +% 0) ; +% end +% +% +% +% function result = ... +% find_and_count_helper(base_folder_path, ... +% is_file_a_keeper_predicate, ... +% folder_predicate_function, ... +% depth, ... +% incoming_count) +% count_so_far = incoming_count ; +% [file_names, is_file_a_folder] = simple_dir(base_folder_path) ; +% file_count = length(file_names) ; +% for i = 1 : file_count , +% file_name = file_names{i} ; +% is_this_file_a_folder = is_file_a_folder(i) ; +% file_path = fullfile(base_folder_path, file_name) ; +% if is_this_file_a_folder , +% % if a folder satisfying predicate function, recurse +% if feval(folder_predicate_function, file_path, depth) , +% count_so_far = ... +% find_and_count_helper(... +% file_path, ... +% is_file_a_keeper_predicate, ... +% folder_predicate_function, ... +% depth+1, ... +% count_so_far) ; +% end +% else +% if feval(is_file_a_keeper_predicate, file_path) , +% count_so_far = count_so_far + 1 ; +% end +% end +% end +% result = count_so_far ; +% end diff --git a/freeze_mouselight_folder.m b/freeze_mouselight_folder.m index f6628ab..30bed72 100644 --- a/freeze_mouselight_folder.m +++ b/freeze_mouselight_folder.m @@ -5,74 +5,96 @@ function freeze_mouselight_folder(... do_run_on_cluster, ... do_try, ... maximum_running_slot_count, ... - submit_host_name) - + submit_host_name, ... + varargin) + + % Handle optional arguments + [do_freezing, do_verification] = parse_keyword_args(... + varargin, ... + 'do_freezing', true, ... + 'do_verification', true) ; + % % Compress full-rez octree tiffs to .mj2, copy all non-tif files over % - % Compress all the full-rez octree tiles to .mj2 - batch_convert_all_tif_files_to_mj2(mj2_output_folder_path, ... - tif_input_folder_path, ... - compression_ratio, ... - do_run_on_cluster, ... - do_try, ... - maximum_running_slot_count, ... - submit_host_name) ; - - % Copy all the non-tiff files over to the tif side - copy_non_tif_files_to_mj2_side(mj2_output_folder_path, ... - tif_input_folder_path) ; + if do_freezing , + % Compress all the full-rez octree tiles to .mj2 + batch_convert_all_tif_files_to_mj2(... + mj2_output_folder_path, ... + tif_input_folder_path, ... + compression_ratio, ... + do_run_on_cluster, ... + do_try, ... + maximum_running_slot_count, ... + submit_host_name) ; - % - % Verify that all that went well - % - - % Use the cluster to check that all compressed .tif's match their .mj2 well - % enough, and write a 'certificate' file to represent that. - fprintf('Doing batch preverification of .tif files...\n') ; - batch_preverify_tif_files_after_mj2_from_tif(mj2_output_folder_path, ... - tif_input_folder_path, ... - do_run_on_cluster, ... - do_try, ... - submit_host_name, ... - maximum_running_slot_count) - fprintf('Done with batch preverification of .tif files.\n') ; - - % Wait a bit to make sure the filesystem changes have propagated - if do_run_on_cluster , - pause(20) ; + % Copy all the non-tiff files over to the mj2 side + copy_non_stack_files_during_freezing(... + mj2_output_folder_path, ... + tif_input_folder_path) ; end - % Check that all the cetificate files that should be there, are there - fprintf('Doing verification of .tif files...\n') ; - verify_tif_files_after_mj2_from_tif(tif_input_folder_path, mj2_output_folder_path) ; - fprintf('Done with verification of .tif files.\n') ; - - % Verify that all the non-tif files exist on the mj2 side - fprintf('Doing verification of non-.tif files...\n') ; - verify_non_tif_files_after_mj2_from_tif(mj2_output_folder_path, tif_input_folder_path) ; - fprintf('Done with verification of non-.tif files.\n') ; - % - % Do a final comparison of .tif files and .mj2 files, just for the heck of it + % Verify that all that went well % - % Count the tifs - final_tif_count = ... - find_and_count(tif_input_folder_path, @does_file_name_end_in_dot_tif) ; - fprintf('Final .tif count in %s is: %d\n', tif_input_folder_path, final_tif_count) ; + if do_verification , + % Use the cluster to check that all compressed .tif's match their .mj2 well + % enough, and write a 'certificate' file to represent that. + fprintf('Doing batch preverification of .tif files...\n') ; + batch_preverify_stack_files_after_freezing(mj2_output_folder_path, ... + tif_input_folder_path, ... + do_run_on_cluster, ... + do_try, ... + submit_host_name, ... + maximum_running_slot_count) + fprintf('Done with batch preverification of .tif files.\n') ; + + % Wait a bit to make sure the filesystem changes have propagated + if do_run_on_cluster , + pause(20) ; + end - % Count the mj2s - final_mj2_count = ... - find_and_count(mj2_output_folder_path, @does_file_name_end_in_dot_mj2) ; - fprintf('Final .mj2 count in %s is: %d\n', mj2_output_folder_path, final_mj2_count) ; + % Check that all the cetificate files that should be there, are there + fprintf('Doing verification of .tif files...\n') ; + verify_tif_files_after_mj2_from_tif(tif_input_folder_path, mj2_output_folder_path) ; + fprintf('Done with verification of .tif files.\n') ; + + % Verify that all the non-tif files exist on the mj2 side + fprintf('Doing verification of non-.tif files...\n') ; + verify_non_stack_files_after_freezing(mj2_output_folder_path, tif_input_folder_path) ; + fprintf('Done with verification of non-.tif files.\n') ; + + % + % Do a final comparison of .tif files and .mj2 files, just for the heck of it + % - % Final check - if final_tif_count ~= final_mj2_count , - error('Final .mj2 count in target folder differs from final .tif count in source folder') ; + % Count the tifs + final_tif_count = ... + find_and_count(tif_input_folder_path, @does_file_name_end_in_dot_tif, @is_not_the_ktx_folder) ; + fprintf('Final .tif count in %s is: %d\n', tif_input_folder_path, final_tif_count) ; + + % Count the mj2s + final_mj2_count = ... + find_and_count(mj2_output_folder_path, @does_file_name_end_in_dot_mj2, @is_not_the_ktx_folder) ; + fprintf('Final .mj2 count in %s is: %d\n', mj2_output_folder_path, final_mj2_count) ; + + % Final check + if final_tif_count ~= final_mj2_count , + error('Final .mj2 count in target folder differs from final .tif count in source folder') ; + end end % Report final status if we get this far - fprintf('Octree compression and verification succeeded!\n') ; + if do_freezing && do_verification , + fprintf('Folder compression and verification succeeded!\n') ; + elseif do_freezing , + fprintf('Folder compression succeeded! No verification performed.\n') ; + elseif do_verification , + fprintf('Folder verification succeeded! (No compression performed.)\n') ; + else + fprintf('Did nothing. Nothing at all.\n') ; + end + end diff --git a/is_leaf_mj2_and_lacks_cert_for_output.m b/is_mj2_and_lacks_cert_for_output.m similarity index 65% rename from is_leaf_mj2_and_lacks_cert_for_output.m rename to is_mj2_and_lacks_cert_for_output.m index eb4f321..93da380 100644 --- a/is_leaf_mj2_and_lacks_cert_for_output.m +++ b/is_mj2_and_lacks_cert_for_output.m @@ -1,6 +1,6 @@ -function result = is_leaf_mj2_and_lacks_cert_for_output(input_file_path, input_root_path, output_root_path) - is_input_file_an_input_leaf_image = is_a_mj2_in_a_folder_with_no_subfolders(input_file_path) ; - if ~is_input_file_an_input_leaf_image , +function result = is_mj2_and_lacks_cert_for_output(input_file_path, input_root_path, output_root_path) + is_input_file_an_mj2 = does_file_name_end_in_dot_mj2(input_file_path) ; + if ~is_input_file_an_mj2 , result = false ; return end @@ -13,6 +13,6 @@ % result = true ; % return % end - does_cert_exist = does_is_similar_to_tif_check_file_exist(input_file_path, input_root_path, output_root_path) ; + does_cert_exist = does_is_similar_to_mj2_check_file_exist(input_file_path, input_root_path, output_root_path) ; result = ~does_cert_exist ; end diff --git a/is_not_the_ktx_folder.m b/is_not_the_ktx_folder.m new file mode 100644 index 0000000..366ea5c --- /dev/null +++ b/is_not_the_ktx_folder.m @@ -0,0 +1,16 @@ +function result = is_not_the_ktx_folder(varargin) + % The last arg should be depth, the rest should be path parts + depth = varargin{nargin} ; + + % We skip the 'ktx' folder at depth zero, if there is one + if depth > 0 , + result = true ; + return + end + + % Must be at depth zero if get here + last_path_part = varargin{nargin-1} ; + [~,folder_name] = fileparts2(last_path_part) ; + do_skip = strcmp(folder_name, 'ktx') ; + result = ~do_skip ; +end diff --git a/mouselight_toolbox b/mouselight_toolbox index 142f553..743b649 160000 --- a/mouselight_toolbox +++ b/mouselight_toolbox @@ -1 +1 @@ -Subproject commit 142f553986e7846988cb25d6f5603767182cff9b +Subproject commit 743b649365ba969a738c8bf4e4b0bddcb898024c diff --git a/thaw_mouselight_folder.m b/thaw_mouselight_folder.m index 8a3ea41..3ebf158 100644 --- a/thaw_mouselight_folder.m +++ b/thaw_mouselight_folder.m @@ -4,77 +4,90 @@ function thaw_mouselight_folder(... do_run_on_cluster, ... do_try, ... maximum_running_slot_count, ... - submit_host_name) - - % - % Phase 1: Decompress all full-rez octree .mj2's to .tif - % + submit_host_name, ... + varargin) - % Compress all the full-rez octree tiles to .mj2 - batch_convert_all_mj2_files_to_tif(... - output_folder_path, ... - input_folder_path, ... - do_run_on_cluster, ... - do_try, ... - maximum_running_slot_count, ... - submit_host_name) ; + % Handle optional arguments + [do_thawing, do_verification] = parse_keyword_args(... + varargin, ... + 'do_thawing', true, ... + 'do_verification', true) ; - % Use the cluster to check that all decompressed .tif's match their .mj2 well - % enough, and write a 'certificate' file to represent that. - fprintf('Doing batch verification of leaf .tif files...\n') ; - batch_verify_leaf_tif_files_after_thawing(... - output_folder_path, ... - input_folder_path, ... - do_run_on_cluster, ... - do_try, ... - submit_host_name, ... - maximum_running_slot_count) - fprintf('Done with batch verification of leaf .tif files.\n') ; + % Stage 1: thawing + if do_thawing , + % Compress all the full-rez octree tiles to .mj2 + batch_convert_all_mj2_files_to_tif(... + output_folder_path, ... + input_folder_path, ... + do_run_on_cluster, ... + do_try, ... + maximum_running_slot_count, ... + submit_host_name) ; - % Wait a bit to make sure the filesystem changes have propagated - if do_run_on_cluster , - pause(20) ; + % Copy all the non-mj2 files over to the tif side + copy_non_stack_files_during_thawing(output_folder_path, ... + input_folder_path) ; end - % Check that all the cetificate files that should be there, are there - fprintf('Checking leaf verification certificates...\n') ; - check_certs_after_verifying_thawing(input_folder_path, output_folder_path) ; - fprintf('Done checking leaf verification certificates.\n') ; - - - % - % Phase 2: Copy all the non-mj2 files over - % - - % Copy all the non-mj2 files over to the tif side - copy_non_mj2_files_to_tif_side(output_folder_path, ... - input_folder_path) ; + % Stage 2: verification + if do_verification , + % Use the cluster to check that all decompressed .tif's match their .mj2 well + % enough, and write a 'certificate' file to represent that. + fprintf('Doing batch verification of output .tif files...\n') ; + batch_verify_tif_files_after_thawing(... + output_folder_path, ... + input_folder_path, ... + do_run_on_cluster, ... + do_try, ... + submit_host_name, ... + maximum_running_slot_count) + fprintf('Done with batch verification of output .tif files.\n') ; + + % Wait a bit to make sure the filesystem changes have propagated + if do_run_on_cluster , + pause(20) ; + end - % Verify that all the non-tif files exist on the mj2 side - fprintf('Doing verification of non-.tif files...\n') ; - verify_non_tif_files_after_thawing(output_folder_path, input_folder_path) ; - fprintf('Done with verification of non-.tif files.\n') ; - + % Check that all the cetificate files that should be there, are there + fprintf('Checking output .tif verification certificates...\n') ; + check_certs_after_verifying_thawing(input_folder_path, output_folder_path) ; + fprintf('Done checking output .tif verification certificates.\n') ; - % - % Do a final comparison of leaf .tif files and .mj2 files, just for the heck of it - % - - % Count the mj2s - final_mj2_count = ... - find_and_count(input_folder_path, @does_file_name_end_in_dot_mj2) ; - fprintf('Final .mj2 count in %s is: %d\n', input_folder_path, final_mj2_count) ; + % Verify that all the non-tif files exist on the mj2 side + fprintf('Doing verification of non-.tif files...\n') ; + verify_non_stack_files_after_thawing(output_folder_path, input_folder_path) ; + fprintf('Done with verification of non-.tif files.\n') ; - % Count the tifs - final_tif_count = ... - find_and_count(output_folder_path, @does_file_name_end_in_dot_tif) ; - fprintf('Final .tif count in %s is: %d\n', output_folder_path, final_tif_count) ; + + % + % Do a final comparison of leaf .tif files and .mj2 files, just for the heck of it + % - % Final check - if final_tif_count ~= final_mj2_count , - error('Final .tif count in target folder differs from final .mj2 count in source folder') ; + % Count the mj2s + final_mj2_count = ... + find_and_count(input_folder_path, @does_file_name_end_in_dot_mj2, @(varargin)(true)) ; + fprintf('Final .mj2 count in %s is: %d\n', input_folder_path, final_mj2_count) ; + + % Count the tifs + final_tif_count = ... + find_and_count(output_folder_path, @does_file_name_end_in_dot_tif, @(varargin)(true)) ; + fprintf('Final .tif count in %s is: %d\n', output_folder_path, final_tif_count) ; + + % Final check + if final_tif_count ~= final_mj2_count , + error('Final .tif count in target folder differs from final .mj2 count in source folder') ; + end end - + % Report final status if we get this far - fprintf('Octree decompression and verification succeeded!\n') ; + if do_thawing && do_verification , + fprintf('Folder decompression and verification succeeded!\n') ; + elseif do_thawing , + fprintf('Folder decompression succeeded! No verification performed.\n') ; + elseif do_verification , + fprintf('Folder verification succeeded! (No decompression performed.)\n') ; + else + fprintf('Did nothing. Nothing at all.\n') ; + end + end diff --git a/verify_non_stack_files_after_freezing.m b/verify_non_stack_files_after_freezing.m new file mode 100644 index 0000000..5321d96 --- /dev/null +++ b/verify_non_stack_files_after_freezing.m @@ -0,0 +1,13 @@ +function verify_non_stack_files_after_freezing(mj2_root_folder_path, tif_root_folder_path) + % Makes sure all the target files are present in the the destination, and have + % the right size, or at least a plausible size in the case of .mj2's. + + function result = verify_single_non_stack_file_wrapper(tif_root_folder_path, tif_folder_relative_path, tif_file_name, depth) %#ok + result = verify_single_non_stack_file(tif_root_folder_path, tif_folder_relative_path, tif_file_name, mj2_root_folder_path) ; + end + + find_and_verify(tif_root_folder_path, ... + @does_file_name_not_end_in_dot_tif, ... + @verify_single_non_stack_file_wrapper, ... + @is_not_the_ktx_folder) ; +end diff --git a/verify_non_stack_files_after_thawing.m b/verify_non_stack_files_after_thawing.m new file mode 100644 index 0000000..89ddc42 --- /dev/null +++ b/verify_non_stack_files_after_thawing.m @@ -0,0 +1,13 @@ +function verify_non_stack_files_after_thawing(output_root_folder_path, input_root_folder_path) + % Makes sure all the output files are present in the the destination, and have + % the same size and md5sum. + + function result = verify_single_non_stack_file_wrapper(input_root_folder_path, base_folder_relative_path, file_name, depth) %#ok + result = verify_single_non_stack_file(input_root_folder_path, base_folder_relative_path, file_name, output_root_folder_path) ; + end + + find_and_verify(input_root_folder_path, ... + @does_file_name_not_end_in_dot_mj2, ... + @verify_single_non_stack_file_wrapper, ... + @(varargin)(true)) ; +end diff --git a/verify_non_tif_files_after_mj2_from_tif.m b/verify_non_tif_files_after_mj2_from_tif.m deleted file mode 100644 index da8b2f9..0000000 --- a/verify_non_tif_files_after_mj2_from_tif.m +++ /dev/null @@ -1,10 +0,0 @@ -function verify_non_tif_files_after_mj2_from_tif(mj2_root_folder_path, tif_root_folder_path) - % Makes sure all the target files are present in the the destination, and have - % the right size, or at least a plausible size in the case of .mj2's. - - find_and_verify(tif_root_folder_path, ... - @does_file_name_not_end_in_dot_tif, ... - @verify_single_non_tif_file_after_mj2_from_tif, ... - tif_root_folder_path, ... - mj2_root_folder_path) ; -end diff --git a/verify_non_tif_files_after_thawing.m b/verify_non_tif_files_after_thawing.m deleted file mode 100644 index 87621b8..0000000 --- a/verify_non_tif_files_after_thawing.m +++ /dev/null @@ -1,10 +0,0 @@ -function verify_non_tif_files_after_thawing(output_root_folder_path, input_root_folder_path) - % Makes sure all the output files are present in the the destination, and have - % the same size and md5sum. - - find_and_verify(input_root_folder_path, ... - @does_file_name_not_end_in_dot_mj2, ... - @verify_single_non_stack_file, ... - input_root_folder_path, ... - output_root_folder_path) ; -end diff --git a/verify_single_non_stack_file.m b/verify_single_non_stack_file.m index ac9c38b..6b1207c 100644 --- a/verify_single_non_stack_file.m +++ b/verify_single_non_stack_file.m @@ -1,7 +1,7 @@ -function result = verify_single_non_stack_file(input_file_path, input_root_folder_path, output_root_folder_path) - % Compare the md5sum of - relative_file_path = relpath(input_file_path, input_root_folder_path) ; - output_file_path = fullfile(output_root_folder_path, relative_file_path) ; +function result = verify_single_non_stack_file(input_root_folder_path, input_base_folder_relative_path, input_file_name, output_root_folder_path) + % Compare the md5sums of the two files + input_file_path = fullfile(input_root_folder_path, input_base_folder_relative_path, input_file_name) ; + output_file_path = fullfile(output_root_folder_path, input_base_folder_relative_path, input_file_name) ; if exist(output_file_path, 'file') , result = are_files_same_size_and_md5sum(input_file_path, output_file_path) ; else diff --git a/verify_single_non_tif_file_after_mj2_from_tif.m b/verify_single_non_tif_file_after_mj2_from_tif.m deleted file mode 100644 index ca7946a..0000000 --- a/verify_single_non_tif_file_after_mj2_from_tif.m +++ /dev/null @@ -1,10 +0,0 @@ -function result = verify_single_non_tif_file_after_mj2_from_tif(tif_side_file_path, tif_root_folder_path, mj2_root_folder_path) - % Compare the md5sum of - relative_file_path = relpath(tif_side_file_path, tif_root_folder_path) ; - mj2_side_file_path = fullfile(mj2_root_folder_path, relative_file_path) ; - if exist(mj2_side_file_path, 'file') , - result = are_files_same_size_and_md5sum(tif_side_file_path, mj2_side_file_path) ; - else - result = false ; - end -end diff --git a/verify_single_tif_file_after_mj2_from_tif.m b/verify_single_tif_file_after_mj2_from_tif.m index 16723ec..360d6ba 100644 --- a/verify_single_tif_file_after_mj2_from_tif.m +++ b/verify_single_tif_file_after_mj2_from_tif.m @@ -1,7 +1,6 @@ -function result = verify_single_tif_file_after_mj2_from_tif(tif_file_path, tif_root_folder_path, mj2_root_folder_path) - relative_file_path_of_tif = relpath(tif_file_path, tif_root_folder_path) ; - relative_file_path_of_mj2 = replace_extension(relative_file_path_of_tif, '.mj2') ; - mj2_file_path = fullfile(mj2_root_folder_path, relative_file_path_of_mj2) ; +function result = verify_single_tif_file_after_mj2_from_tif(tif_root_folder_path, tif_folder_relative_path, tif_file_name, mj2_root_folder_path) %#ok + file_name_of_mj2 = replace_extension(tif_file_name, '.mj2') ; + mj2_file_path = fullfile(mj2_root_folder_path, tif_folder_relative_path, file_name_of_mj2) ; check_file_path = horzcat(mj2_file_path, '.is-similar-to-tif') ; result = logical( exist(check_file_path, 'file') ) ; end diff --git a/verify_tif_files_after_mj2_from_tif.m b/verify_tif_files_after_mj2_from_tif.m index 8462979..992fe2b 100644 --- a/verify_tif_files_after_mj2_from_tif.m +++ b/verify_tif_files_after_mj2_from_tif.m @@ -1,10 +1,14 @@ function verify_tif_files_after_mj2_from_tif(tif_root_folder_path, mj2_root_folder_path) % Makes sure there's a .similar-mj2-exists for each .tif file in % tif_root_folder_path. + + function result = verify_single_tif_file_after_mj2_from_tif_wrapper(tif_root_folder_path, tif_folder_relative_path, tif_file_name, depth) %#ok + result = verify_single_tif_file_after_mj2_from_tif(tif_root_folder_path, tif_folder_relative_path, tif_file_name, mj2_root_folder_path) ; + end + find_and_verify(... tif_root_folder_path, ... @does_file_name_end_in_dot_tif, ... - @verify_single_tif_file_after_mj2_from_tif, ... - tif_root_folder_path, ... - mj2_root_folder_path) ; + @verify_single_tif_file_after_mj2_from_tif_wrapper, ... + @is_not_the_ktx_folder) ; end