From d3b4017e9f85761e7ae78f3d33191d7bddc98fef Mon Sep 17 00:00:00 2001 From: "Adam L. Taylor" Date: Thu, 18 May 2023 14:38:20 -0400 Subject: [PATCH] master: Replaced with code from octree repo, b/c more modern --- .gitmodules | 3 + are_files_same_size_and_md5sum.m | 9 ++ batch_convert_all_mj2_files_to_tif.m | 22 +++++ batch_convert_all_tif_files_to_mj2.m | 23 +++++ batch_convert_leaf_mj2_files_to_tif.m | 22 +++++ batch_convert_leaf_tif_files_to_mj2.m | 23 +++++ ...h_preverify_tif_files_after_mj2_from_tif.m | 21 ++-- ...if_files_for_2019_03_26_frozen_raw_tiles.m | 4 - batch_verify_leaf_tif_files_after_thawing.m | 23 +++++ check_certs_after_verifying_thawing.m | 10 ++ check_single_cert_after_verifying_thawing.m | 7 ++ compare_similarity_measures.m | 20 ---- copy_file_for_find_and_batch.m | 8 ++ copy_non_mj2_files_to_tif_side.m | 22 +++++ copy_non_tif_files_to_mj2_side.m | 22 +++++ create_octree_test_folder.m | 26 +++++ does_folder_have_subfolders.m | 4 + does_need_preverification_after_freezing.m | 3 +- find_and_count.m | 37 +++++++ fraction_power_explained.m | 22 ----- fraction_variance_explained.m | 30 +++--- freeze_2018_03_09_rendered.m | 7 -- freeze_2018_07_02.m | 7 -- freeze_2018_08_01.m | 7 -- freeze_2018_10_01.m | 7 -- freeze_2018_12_01.m | 8 -- freeze_2019_03_26_raw_tiles.m | 7 -- freeze_2019_04_17.m | 7 -- freeze_2019_05_27.m | 7 -- freeze_2019_07_19.m | 7 -- freeze_2019_08_08.m | 7 -- freeze_2019_08_22.m | 8 -- freeze_2019_09_06.m | 7 -- freeze_2019_10_04.m | 8 -- freeze_2019_10_31.m | 8 -- freeze_2019_12_25.m | 7 -- freeze_2020_01_13.m | 8 -- freeze_2020_01_23.m | 9 -- freeze_2020_02_01.m | 8 -- freeze_2020_02_24.m | 9 -- freeze_2020_04_15.m | 9 -- freeze_2020_09_15.m | 9 -- freeze_2020_10_16.m | 8 -- freeze_2020_10_16_driver_script.sh | 3 - freeze_2020_11_26.m | 9 -- freeze_2020_11_26_compression_test.m | 8 -- freeze_2020_12_31.m | 8 -- freeze_2020_12_31_driver_script.sh | 3 - freeze_2021_02_01.m | 8 -- freeze_2021_02_05.m | 9 -- freeze_2021_03_17.m | 9 -- freeze_2021_04_10.m | 9 -- freeze_2021_05_22.m | 9 -- freeze_2021_06_06.m | 9 -- freeze_2021_07_20.m | 7 -- freeze_2021_07_20_alt.m | 7 -- freeze_2021_07_20_debug.m | 7 -- freeze_2021_07_20b.m | 8 -- freeze_2021_07_20b_driver_script.sh | 3 - freeze_2021_08_17.m | 9 -- freeze_2021_09_16.m | 8 -- freeze_2021_10_07.m | 8 -- freeze_2021_10_19.m | 8 -- freeze_2021_12_17.m | 8 -- freeze_2022_02_01.m | 8 -- freeze_2022_03_23.m | 8 -- freeze_2022_04_25.m | 8 -- freeze_2022_05_10.m | 8 -- freeze_2022_05_30.m | 11 ++- freeze_mouselight_folder.m | 78 +++++++++++++++ freeze_nrs_test_tf.m | 9 -- freeze_octree_2023_02_23.m | 17 ++++ freeze_sample.m | 43 -------- freeze_temp.m | 8 -- is_a_mj2_and_target_missing.m | 12 +++ is_a_mj2_in_a_folder_with_no_subfolders.m | 9 ++ ...er_with_no_subfolders_and_target_missing.m | 12 +++ is_a_non_mj2_and_target_missing.m | 13 +++ is_a_non_tif_and_target_missing.m | 13 +++ is_a_tif_and_target_present.m | 12 +++ is_a_tif_in_a_folder_with_no_subfolders.m | 9 ++ ...er_with_no_subfolders_and_target_missing.m | 12 +++ ...er_with_no_subfolders_and_target_present.m | 12 +++ is_file_below_depth_limit.m | 10 ++ is_leaf_mj2_and_lacks_cert_for_output.m | 18 ++++ is_leaf_mj2_and_lacks_either_output_or_cert.m | 20 ++++ is_mj2_same_as_tif.m | 9 -- is_mj2_similar_to_tif.m | 23 ----- is_mj2_similar_to_tif_from_paths.m | 23 ----- mj2_from_tif.m | 92 ----------------- mj2_from_tif_single.m | 1 - mj2_from_tif_single_for_find_and_batch.m | 9 ++ mj2_from_tif_single_with_temps.m | 10 -- modpath.m | 4 +- mouselight_toolbox | 1 + ...erify_single_tif_file_after_mj2_from_tif.m | 2 +- quickhydrate_2018_03_09.m | 5 - rebuild_tif_octree_from_leaves.m | 5 + test_everything.m | 9 ++ test_freezing_locally.m | 13 --- test_freezing_on_cluster.m | 14 --- test_something.m | 49 +++++++++ thaw_2017_10_31_sample.m | 3 - thaw_2017_12_19_sample.m | 6 +- thaw_mouselight_folder.m | 80 +++++++++++++++ tif_from_mj2_single_for_find_and_batch.m | 9 ++ uncompress_compression_examples.m | 5 - validate_fpe.m | 96 ------------------ validate_fve.m | 99 ------------------- verify_file_sizes_after_mj2_from_tif.m | 73 -------------- ...le_sizes_for_2019_03_26_frozen_raw_tiles.m | 5 - verify_files_after_mj2_from_tif.m | 74 -------------- verify_frozen_2018_07_02.m | 5 - verify_frozen_2019_04_17.m | 5 - verify_frozen_files.m | 17 ---- verify_non_tif_files_after_thawing.m | 10 ++ ...if_files_for_2019_03_26_frozen_raw_tiles.m | 4 - verify_single_non_stack_file.m | 10 ++ ...y_single_non_tif_file_after_mj2_from_tif.m | 14 ++- verify_single_small_file_after_mj2_from_tif.m | 12 --- verify_single_tif_file_after_thawing.m | 29 ++++++ verify_tif_files_after_mj2_from_tif.m | 14 +-- ...if_files_for_2019_03_26_frozen_raw_tiles.m | 4 - voxelwise_mean_relative_error_with_guard.m | 16 --- why_are_fve_values_so_low.m | 53 ---------- 125 files changed, 783 insertions(+), 1166 deletions(-) create mode 100644 are_files_same_size_and_md5sum.m create mode 100644 batch_convert_all_mj2_files_to_tif.m create mode 100644 batch_convert_all_tif_files_to_mj2.m create mode 100644 batch_convert_leaf_mj2_files_to_tif.m create mode 100644 batch_convert_leaf_tif_files_to_mj2.m delete mode 100644 batch_preverify_tif_files_for_2019_03_26_frozen_raw_tiles.m create mode 100644 batch_verify_leaf_tif_files_after_thawing.m create mode 100644 check_certs_after_verifying_thawing.m create mode 100644 check_single_cert_after_verifying_thawing.m delete mode 100644 compare_similarity_measures.m create mode 100644 copy_file_for_find_and_batch.m create mode 100644 copy_non_mj2_files_to_tif_side.m create mode 100644 copy_non_tif_files_to_mj2_side.m create mode 100644 create_octree_test_folder.m create mode 100644 does_folder_have_subfolders.m create mode 100644 find_and_count.m delete mode 100644 fraction_power_explained.m delete mode 100644 freeze_2018_03_09_rendered.m delete mode 100644 freeze_2018_07_02.m delete mode 100644 freeze_2018_08_01.m delete mode 100644 freeze_2018_10_01.m delete mode 100644 freeze_2018_12_01.m delete mode 100644 freeze_2019_03_26_raw_tiles.m delete mode 100644 freeze_2019_04_17.m delete mode 100644 freeze_2019_05_27.m delete mode 100644 freeze_2019_07_19.m delete mode 100644 freeze_2019_08_08.m delete mode 100644 freeze_2019_08_22.m delete mode 100644 freeze_2019_09_06.m delete mode 100644 freeze_2019_10_04.m delete mode 100644 freeze_2019_10_31.m delete mode 100644 freeze_2019_12_25.m delete mode 100644 freeze_2020_01_13.m delete mode 100644 freeze_2020_01_23.m delete mode 100644 freeze_2020_02_01.m delete mode 100644 freeze_2020_02_24.m delete mode 100644 freeze_2020_04_15.m delete mode 100644 freeze_2020_09_15.m delete mode 100644 freeze_2020_10_16.m delete mode 100644 freeze_2020_10_16_driver_script.sh delete mode 100644 freeze_2020_11_26.m delete mode 100644 freeze_2020_11_26_compression_test.m delete mode 100644 freeze_2020_12_31.m delete mode 100644 freeze_2020_12_31_driver_script.sh delete mode 100644 freeze_2021_02_01.m delete mode 100644 freeze_2021_02_05.m delete mode 100644 freeze_2021_03_17.m delete mode 100644 freeze_2021_04_10.m delete mode 100644 freeze_2021_05_22.m delete mode 100644 freeze_2021_06_06.m delete mode 100644 freeze_2021_07_20.m delete mode 100644 freeze_2021_07_20_alt.m delete mode 100644 freeze_2021_07_20_debug.m delete mode 100644 freeze_2021_07_20b.m delete mode 100644 freeze_2021_07_20b_driver_script.sh delete mode 100644 freeze_2021_08_17.m delete mode 100644 freeze_2021_09_16.m delete mode 100644 freeze_2021_10_07.m delete mode 100644 freeze_2021_10_19.m delete mode 100644 freeze_2021_12_17.m delete mode 100644 freeze_2022_02_01.m delete mode 100644 freeze_2022_03_23.m delete mode 100644 freeze_2022_04_25.m delete mode 100644 freeze_2022_05_10.m create mode 100644 freeze_mouselight_folder.m delete mode 100644 freeze_nrs_test_tf.m create mode 100644 freeze_octree_2023_02_23.m delete mode 100644 freeze_sample.m delete mode 100644 freeze_temp.m create mode 100644 is_a_mj2_and_target_missing.m create mode 100644 is_a_mj2_in_a_folder_with_no_subfolders.m create mode 100644 is_a_mj2_in_a_folder_with_no_subfolders_and_target_missing.m create mode 100644 is_a_non_mj2_and_target_missing.m create mode 100644 is_a_non_tif_and_target_missing.m create mode 100644 is_a_tif_and_target_present.m create mode 100644 is_a_tif_in_a_folder_with_no_subfolders.m create mode 100644 is_a_tif_in_a_folder_with_no_subfolders_and_target_missing.m create mode 100644 is_a_tif_in_a_folder_with_no_subfolders_and_target_present.m create mode 100644 is_file_below_depth_limit.m create mode 100644 is_leaf_mj2_and_lacks_cert_for_output.m create mode 100644 is_leaf_mj2_and_lacks_either_output_or_cert.m delete mode 100644 is_mj2_same_as_tif.m delete mode 100644 is_mj2_similar_to_tif.m delete mode 100644 is_mj2_similar_to_tif_from_paths.m delete mode 100644 mj2_from_tif.m create mode 100644 mj2_from_tif_single_for_find_and_batch.m delete mode 100644 mj2_from_tif_single_with_temps.m create mode 160000 mouselight_toolbox delete mode 100644 quickhydrate_2018_03_09.m create mode 100644 rebuild_tif_octree_from_leaves.m create mode 100644 test_everything.m delete mode 100644 test_freezing_locally.m delete mode 100644 test_freezing_on_cluster.m create mode 100644 test_something.m delete mode 100644 thaw_2017_10_31_sample.m create mode 100644 thaw_mouselight_folder.m create mode 100644 tif_from_mj2_single_for_find_and_batch.m delete mode 100644 uncompress_compression_examples.m delete mode 100644 validate_fpe.m delete mode 100644 validate_fve.m delete mode 100644 verify_file_sizes_after_mj2_from_tif.m delete mode 100644 verify_file_sizes_for_2019_03_26_frozen_raw_tiles.m delete mode 100644 verify_files_after_mj2_from_tif.m delete mode 100644 verify_frozen_2018_07_02.m delete mode 100644 verify_frozen_2019_04_17.m delete mode 100644 verify_frozen_files.m create mode 100644 verify_non_tif_files_after_thawing.m delete mode 100644 verify_non_tif_files_for_2019_03_26_frozen_raw_tiles.m create mode 100644 verify_single_non_stack_file.m delete mode 100644 verify_single_small_file_after_mj2_from_tif.m create mode 100644 verify_single_tif_file_after_thawing.m delete mode 100644 verify_tif_files_for_2019_03_26_frozen_raw_tiles.m delete mode 100644 voxelwise_mean_relative_error_with_guard.m delete mode 100644 why_are_fve_values_so_low.m diff --git a/.gitmodules b/.gitmodules index e69de29..641eb9b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "mouselight_toolbox"] + path = mouselight_toolbox + url = https://github.com/MouseLightProject/mouselight_toolbox diff --git a/are_files_same_size_and_md5sum.m b/are_files_same_size_and_md5sum.m new file mode 100644 index 0000000..a24f981 --- /dev/null +++ b/are_files_same_size_and_md5sum.m @@ -0,0 +1,9 @@ +function result = are_files_same_size_and_md5sum(file_name_1, file_name_2) + file_size_1 = get_file_size(file_name_1) ; + file_size_2 = get_file_size(file_name_2) ; + if file_size_1 ~= file_size_2 , + result = false ; + return + end + result = are_md5s_the_same(file_name_1, file_name_2) ; +end diff --git a/batch_convert_all_mj2_files_to_tif.m b/batch_convert_all_mj2_files_to_tif.m new file mode 100644 index 0000000..850483d --- /dev/null +++ b/batch_convert_all_mj2_files_to_tif.m @@ -0,0 +1,22 @@ +function batch_convert_all_mj2_files_to_tif(output_folder_name, ... + input_folder_name, ... + do_submit, ... + do_try, ... + maximum_running_slot_count, ... + submit_host_name) + % Find all .mj2 files in folders with no subfolders, convert them to .tif + + slots_per_job = 1 ; + bsub_options = '-P mouselight -W 59 -J thaw' ; + find_and_batch(input_folder_name, ... + @is_a_mj2_and_target_missing, ... + @tif_from_mj2_single_for_find_and_batch, ... + 'do_submit', do_submit, ... + 'do_try', do_try, ... + 'submit_host_name', submit_host_name, ... + 'maximum_running_slot_count', maximum_running_slot_count, ... + 'slots_per_job', slots_per_job, ... + 'bsub_options', bsub_options, ... + 'predicate_extra_args', {input_folder_name, output_folder_name}, ... + 'batch_function_extra_args', {input_folder_name, output_folder_name}) ; +end diff --git a/batch_convert_all_tif_files_to_mj2.m b/batch_convert_all_tif_files_to_mj2.m new file mode 100644 index 0000000..6700df4 --- /dev/null +++ b/batch_convert_all_tif_files_to_mj2.m @@ -0,0 +1,23 @@ +function batch_convert_all_tif_files_to_mj2(mj2_folder_name, ... + tif_folder_name, ... + compression_ratio, ... + do_submit, ... + do_try, ... + maximum_running_slot_count, ... + submit_host_name) + % Find all .tif files in folder tree, convert them to .mj2 + + slots_per_job = 1 ; + bsub_options = '-P mouselight -W 59 -J freeze' ; + find_and_batch(tif_folder_name, ... + @does_file_name_end_in_dot_tif_and_mj2_is_missing, ... + @mj2_from_tif_single_for_find_and_batch, ... + 'do_submit', do_submit, ... + 'do_try', do_try, ... + 'submit_host_name', submit_host_name, ... + 'maximum_running_slot_count', maximum_running_slot_count, ... + 'slots_per_job', slots_per_job, ... + 'bsub_options', bsub_options, ... + 'predicate_extra_args', {tif_folder_name, mj2_folder_name}, ... + 'batch_function_extra_args', {tif_folder_name, mj2_folder_name, compression_ratio}) ; +end diff --git a/batch_convert_leaf_mj2_files_to_tif.m b/batch_convert_leaf_mj2_files_to_tif.m new file mode 100644 index 0000000..097bbeb --- /dev/null +++ b/batch_convert_leaf_mj2_files_to_tif.m @@ -0,0 +1,22 @@ +function batch_convert_leaf_mj2_files_to_tif(output_folder_name, ... + input_folder_name, ... + do_submit, ... + do_try, ... + maximum_running_slot_count, ... + submit_host_name) + % Find all .mj2 files in folders with no subfolders, convert them to .tif + + slots_per_job = 1 ; + bsub_options = '-P mouselight -W 59 -J thaw' ; + find_and_batch(input_folder_name, ... + @is_a_mj2_in_a_folder_with_no_subfolders_and_target_missing, ... + @tif_from_mj2_single_for_find_and_batch, ... + 'do_submit', do_submit, ... + 'do_try', do_try, ... + 'submit_host_name', submit_host_name, ... + 'maximum_running_slot_count', maximum_running_slot_count, ... + 'slots_per_job', slots_per_job, ... + 'bsub_options', bsub_options, ... + 'predicate_extra_args', {input_folder_name, output_folder_name}, ... + 'batch_function_extra_args', {input_folder_name, output_folder_name}) ; +end diff --git a/batch_convert_leaf_tif_files_to_mj2.m b/batch_convert_leaf_tif_files_to_mj2.m new file mode 100644 index 0000000..647df7c --- /dev/null +++ b/batch_convert_leaf_tif_files_to_mj2.m @@ -0,0 +1,23 @@ +function batch_convert_leaf_tif_files_to_mj2(mj2_folder_name, ... + tif_folder_name, ... + compression_ratio, ... + do_submit, ... + do_try, ... + maximum_running_slot_count, ... + submit_host_name) + % Find all .tif files in folders with no subfolders, convert them to .mj2 + + slots_per_job = 1 ; + bsub_options = '-P mouselight -W 59 -J freeze' ; + find_and_batch(tif_folder_name, ... + @is_a_tif_in_a_folder_with_no_subfolders_and_target_missing, ... + @mj2_from_tif_single_for_find_and_batch, ... + 'do_submit', do_submit, ... + 'do_try', do_try, ... + 'submit_host_name', submit_host_name, ... + 'maximum_running_slot_count', maximum_running_slot_count, ... + 'slots_per_job', slots_per_job, ... + 'bsub_options', bsub_options, ... + 'predicate_extra_args', {tif_folder_name, mj2_folder_name}, ... + 'batch_function_extra_args', {tif_folder_name, mj2_folder_name, compression_ratio}) ; +end diff --git a/batch_preverify_tif_files_after_mj2_from_tif.m b/batch_preverify_tif_files_after_mj2_from_tif.m index 2ce7362..b655d14 100644 --- a/batch_preverify_tif_files_after_mj2_from_tif.m +++ b/batch_preverify_tif_files_after_mj2_from_tif.m @@ -1,4 +1,9 @@ -function batch_preverify_tif_files_after_mj2_from_tif(mj2_folder_name, tif_folder_name, do_submit, maximum_running_slot_count) +function batch_preverify_tif_files_after_mj2_from_tif(mj2_folder_name, ... + tif_folder_name, ... + do_submit, ... + do_try, ... + submit_host_name, ... + maximum_running_slot_count) % Makes sure all the big target files are present in the the destination, and % are similar to their source files. @@ -7,10 +12,12 @@ function batch_preverify_tif_files_after_mj2_from_tif(mj2_folder_name, tif_folde find_and_batch(tif_folder_name, ... @does_need_preverification_after_freezing, ... @preverify_single_tif_file_after_mj2_from_tif, ... - do_submit, ... - maximum_running_slot_count, ... - slots_per_job, ... - bsub_options, ... - tif_folder_name, ... - mj2_folder_name) ; + 'do_submit', do_submit, ... + 'do_try', do_try, ... + 'submit_host_name', submit_host_name, ... + 'maximum_running_slot_count', maximum_running_slot_count, ... + 'slots_per_job', slots_per_job, ... + 'bsub_options', bsub_options, ... + 'predicate_extra_args', {tif_folder_name, mj2_folder_name}, ... + 'batch_function_extra_args', {tif_folder_name, mj2_folder_name}) ; end diff --git a/batch_preverify_tif_files_for_2019_03_26_frozen_raw_tiles.m b/batch_preverify_tif_files_for_2019_03_26_frozen_raw_tiles.m deleted file mode 100644 index cd35d3e..0000000 --- a/batch_preverify_tif_files_for_2019_03_26_frozen_raw_tiles.m +++ /dev/null @@ -1,4 +0,0 @@ -sample_date = '2019-03-26' ; -mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -batch_verify_tif_files_after_mj2_from_tif(mj2_output_folder_path, tif_input_folder_path) diff --git a/batch_verify_leaf_tif_files_after_thawing.m b/batch_verify_leaf_tif_files_after_thawing.m new file mode 100644 index 0000000..17e0ec8 --- /dev/null +++ b/batch_verify_leaf_tif_files_after_thawing.m @@ -0,0 +1,23 @@ +function batch_verify_leaf_tif_files_after_thawing(output_folder_name, ... + input_folder_name, ... + do_submit, ... + do_try, ... + submit_host_name, ... + maximum_running_slot_count) + % Makes sure all the leaf image files are present in the the destination, and + % are similar to their source files. + + slots_per_job = 4 ; + bsub_options = '-P mouselight -W 59 -J verify-thawed-leaves' ; + find_and_batch(input_folder_name, ... + @is_leaf_mj2_and_lacks_cert_for_output, ... + @verify_single_tif_file_after_thawing, ... + 'do_submit', do_submit, ... + 'do_try', do_try, ... + 'submit_host_name', submit_host_name, ... + 'maximum_running_slot_count', maximum_running_slot_count, ... + 'slots_per_job', slots_per_job, ... + 'bsub_options', bsub_options, ... + 'predicate_extra_args', {input_folder_name, output_folder_name}, ... + 'batch_function_extra_args', {input_folder_name, output_folder_name}) ; +end diff --git a/check_certs_after_verifying_thawing.m b/check_certs_after_verifying_thawing.m new file mode 100644 index 0000000..40b5298 --- /dev/null +++ b/check_certs_after_verifying_thawing.m @@ -0,0 +1,10 @@ +function check_certs_after_verifying_thawing(input_root_folder_path, output_root_folder_path) + % Makes sure there's a .similar-tif-exists for each .mj2 file in + % input_root_folder_path. + find_and_verify(... + input_root_folder_path, ... + @is_a_mj2_in_a_folder_with_no_subfolders, ... + @check_single_cert_after_verifying_thawing, ... + input_root_folder_path, ... + output_root_folder_path) ; +end diff --git a/check_single_cert_after_verifying_thawing.m b/check_single_cert_after_verifying_thawing.m new file mode 100644 index 0000000..5828ba9 --- /dev/null +++ b/check_single_cert_after_verifying_thawing.m @@ -0,0 +1,7 @@ +function result = check_single_cert_after_verifying_thawing(input_file_path, input_root_folder_path, output_root_folder_path) + input_relative_file_path = relpath(input_file_path, input_root_folder_path) ; + output_relative_file_path = replace_extension(input_relative_file_path, '.tif') ; + output_file_path = fullfile(output_root_folder_path, output_relative_file_path) ; + cert_file_path = horzcat(output_file_path, '.is-similar-to-mj2') ; + result = logical( exist(cert_file_path, 'file') ) ; +end diff --git a/compare_similarity_measures.m b/compare_similarity_measures.m deleted file mode 100644 index c37c6d7..0000000 --- a/compare_similarity_measures.m +++ /dev/null @@ -1,20 +0,0 @@ -tif_file_name = '01018-ngc.0.tif' ; -mj2_file_name = '01018-ngc.0.mj2' ; - -tic_id = tic() ; -tif_stack = read_16bit_grayscale_tif(tif_file_name) ; -mj2_stack = read_16bit_grayscale_mj2(mj2_file_name) ; -toc(tic_id) - -tic_id = tic() ; -ssim_value = ssim(mj2_stack, tif_stack) -toc(tic_id) - -tic_id = tic() ; -vmre_value = voxelwise_mean_relative_error_with_guard(mj2_stack, tif_stack, 1/256) -toc(tic_id) - -tic_id = tic() ; -fpe_value = fraction_power_explained(mj2_stack, tif_stack) -toc(tic_id) - diff --git a/copy_file_for_find_and_batch.m b/copy_file_for_find_and_batch.m new file mode 100644 index 0000000..fa840a6 --- /dev/null +++ b/copy_file_for_find_and_batch.m @@ -0,0 +1,8 @@ +function copy_file_for_find_and_batch(tif_side_file_path, tif_root_folder_name, mj2_root_folder_name, varargin) + relative_file_path_of_tif_side_file = relpath(tif_side_file_path, tif_root_folder_name) ; + relative_file_path_of_mj2_side_file = relative_file_path_of_tif_side_file ; + mj2_side_file_path = fullfile(mj2_root_folder_name, relative_file_path_of_mj2_side_file) ; + mj2_side_folder_path = fileparts2(mj2_side_file_path) ; + ensure_folder_exists(mj2_side_folder_path) ; + copyfile(tif_side_file_path, mj2_side_file_path) ; +end diff --git a/copy_non_mj2_files_to_tif_side.m b/copy_non_mj2_files_to_tif_side.m new file mode 100644 index 0000000..f5dc20a --- /dev/null +++ b/copy_non_mj2_files_to_tif_side.m @@ -0,0 +1,22 @@ +function copy_non_mj2_files_to_tif_side(output_folder_name, ... + input_folder_name) + % Find all non-.mj2 files, copy them over to the tif side + + do_submit = false ; + do_try = true ; + maximum_running_slot_count = 400 ; % doesn't matter + slots_per_job = 1 ; % doesn't matter + bsub_options = '-P mouselight -W 59 -J copy' ; % doesn't matter + submit_host_name = '' ; + find_and_batch(input_folder_name, ... + @is_a_non_mj2_and_target_missing, ... + @copy_file_for_find_and_batch, ... + 'do_submit', do_submit, ... + 'do_try', do_try, ... + 'submit_host_name', submit_host_name, ... + 'maximum_running_slot_count', maximum_running_slot_count, ... + 'slots_per_job', slots_per_job, ... + 'bsub_options', bsub_options, ... + 'predicate_extra_args', {input_folder_name, output_folder_name}, ... + 'batch_function_extra_args', {input_folder_name, output_folder_name}) ; +end diff --git a/copy_non_tif_files_to_mj2_side.m b/copy_non_tif_files_to_mj2_side.m new file mode 100644 index 0000000..f7b3feb --- /dev/null +++ b/copy_non_tif_files_to_mj2_side.m @@ -0,0 +1,22 @@ +function copy_non_tif_files_to_mj2_side(mj2_folder_name, ... + tif_folder_name) + % Find all non-.tif files, copy them over to the mj2 side + + do_submit = false ; + do_try = true ; + maximum_running_slot_count = 400 ; % doesn't matter + slots_per_job = 1 ; % doesn't matter + bsub_options = '-P mouselight -W 59 -J copy' ; % doesn't matter + submit_host_name = '' ; + find_and_batch(tif_folder_name, ... + @is_a_non_tif_and_target_missing, ... + @copy_file_for_find_and_batch, ... + 'do_submit', do_submit, ... + 'do_try', do_try, ... + 'submit_host_name', submit_host_name, ... + 'maximum_running_slot_count', maximum_running_slot_count, ... + 'slots_per_job', slots_per_job, ... + 'bsub_options', bsub_options, ... + 'predicate_extra_args', {tif_folder_name, mj2_folder_name} , ... + 'batch_function_extra_args', {tif_folder_name, mj2_folder_name} ) ; +end diff --git a/create_octree_test_folder.m b/create_octree_test_folder.m new file mode 100644 index 0000000..d581e97 --- /dev/null +++ b/create_octree_test_folder.m @@ -0,0 +1,26 @@ +output_folder_name = '/groups/mousebrainmicro/mousebrainmicro/scripts/octree-freezing/octree-rebuilding-test-input-folder' ; +input_folder_name = '/nrs/mouselight/SAMPLES/2023-02-23' ; +max_depth = 2 ; +do_submit = true ; +do_try = false ; +maximum_running_slot_count = 400 ; +slots_per_job = 1 ; +bsub_options = '-P mouselight -W 59 -J copy' ; +submit_host_name = 'login2.int.janelia.org' ; + +% Delete the test output folder +reset_for_test(output_folder_name) ; + +% Do the copy +find_and_batch(input_folder_name, ... + @(file_path)(true), ... + @copy_file_for_find_and_batch, ... + 'do_submit', do_submit, ... + 'do_try', do_try, ... + 'submit_host_name', submit_host_name, ... + 'maximum_running_slot_count', maximum_running_slot_count, ... + 'slots_per_job', slots_per_job, ... + 'bsub_options', bsub_options, ... + 'max_depth', max_depth, ... + 'predicate_extra_args', cell(1,0), ... + 'batch_function_extra_args', {input_folder_name, output_folder_name}) ; diff --git a/does_folder_have_subfolders.m b/does_folder_have_subfolders.m new file mode 100644 index 0000000..fef4b09 --- /dev/null +++ b/does_folder_have_subfolders.m @@ -0,0 +1,4 @@ +function result = does_folder_have_subfolders(folder_name) + [name_from_thing_index, is_folder_from_thing_index] = simple_dir(folder_name) ; %#ok + result = any(is_folder_from_thing_index) ; +end diff --git a/does_need_preverification_after_freezing.m b/does_need_preverification_after_freezing.m index cb99f5d..f4dc117 100644 --- a/does_need_preverification_after_freezing.m +++ b/does_need_preverification_after_freezing.m @@ -1,5 +1,4 @@ function result = does_need_preverification_after_freezing(tif_file_path, tif_root_path, mj2_root_path) - [~,file_name] = fileparts2(tif_file_path) ; - result = does_file_name_end_in_dot_tif(file_name) && ... + result = is_a_tif_and_target_present(tif_file_path, tif_root_path, mj2_root_path) && ... ~does_is_similar_to_tif_check_file_exist(tif_file_path, tif_root_path, mj2_root_path) ; end diff --git a/find_and_count.m b/find_and_count.m new file mode 100644 index 0000000..c803803 --- /dev/null +++ b/find_and_count.m @@ -0,0 +1,37 @@ +function result = find_and_count(base_folder_path, is_file_a_keeper_predicate, varargin) + result = ... + find_and_list_helper(base_folder_path, ... + is_file_a_keeper_predicate, ... + 0, ... + varargin{:}) ; +end + + + +function result = ... + find_and_list_helper(base_folder_path, ... + is_file_a_keeper_predicate, ... + incoming_count, ... + varargin) + count_so_far = incoming_count ; + [file_names, is_file_a_folder] = simple_dir(base_folder_path) ; + file_count = length(file_names) ; + for i = 1 : file_count , + file_name = file_names{i} ; + is_this_file_a_folder = is_file_a_folder(i) ; + file_path = fullfile(base_folder_path, file_name) ; + if is_this_file_a_folder , + % if a folder, recurse + count_so_far = ... + find_and_list_helper(file_path, ... + is_file_a_keeper_predicate, ... + count_so_far, ... + varargin{:}) ; + else + if feval(is_file_a_keeper_predicate, file_path, varargin{:}) , + count_so_far = count_so_far + 1 ; + end + end + end + result = count_so_far ; +end diff --git a/fraction_power_explained.m b/fraction_power_explained.m deleted file mode 100644 index 6c8f31f..0000000 --- a/fraction_power_explained.m +++ /dev/null @@ -1,22 +0,0 @@ -function result = fraction_power_explained(mj2_stack, tif_stack) - if isequal(class(mj2_stack), class(tif_stack)) && isequal(size(mj2_stack), size(tif_stack)) , - if all(all(all(mj2_stack==tif_stack))) , - result = 1 ; - else - mj2_stack_serial = double(mj2_stack(:))/(2^16-1) ; - tif_stack_serial = double(tif_stack(:))/(2^16-1) ; - - error_power = mean( (mj2_stack_serial-tif_stack_serial).^2 ) ; - image_power = mean( tif_stack_serial.^2 ) ; - - - % PSNR is Imax^2 / MSE, where Imax is the max possible image intensity, which is - % 1 in this case because we divided by 2^16-1. So PSNR = 1/MSE. So the "PNSR" - % is just MSE. So 1-mse is the fraction of variance explained - - result = 1 - error_power/image_power ; - end - else - result = 0 ; - end -end diff --git a/fraction_variance_explained.m b/fraction_variance_explained.m index 7a226ad..2601f7d 100644 --- a/fraction_variance_explained.m +++ b/fraction_variance_explained.m @@ -1,21 +1,27 @@ -function result = fraction_variance_explained(mj2_stack, tif_stack) - if isequal(class(mj2_stack), class(tif_stack)) && isequal(size(mj2_stack), size(tif_stack)) , - if all(all(all(mj2_stack==tif_stack))) , +function result = fraction_variance_explained(output_stack, input_stack) + % Compute the fraction of the variance in the input stack which is explained + % by the output stack. This is just a gross check that the output is close to + % the input. + if isequal(class(output_stack), class(input_stack)) && isequal(size(output_stack), size(input_stack)) , + if all(all(all(output_stack==input_stack))) , result = 1 ; else - mj2_stack_serial = double(mj2_stack(:)) ; - tif_stack_serial = double(tif_stack(:)) ; - tif_mean = mean(tif_stack_serial) + output_stack_serial = double(output_stack(:)) ; + input_stack_serial = double(input_stack(:)) ; + input_mean = mean(input_stack_serial) ; - residual_variance = mean( (mj2_stack_serial-tif_stack_serial).^2 ) - sqrt_residual_variance = sqrt(residual_variance) - total_variance = mean( (tif_stack_serial-tif_mean).^2 ) - sqrt_total_variance = sqrt(total_variance) + residual_variance = mean( (output_stack_serial-input_stack_serial).^2 ) ; + %sqrt_residual_variance = sqrt(residual_variance) + total_variance = mean( (input_stack_serial-input_mean).^2 ) ; + %sqrt_total_variance = sqrt(total_variance) if residual_variance == 0 , - result = 1 ; % want this to be so even if total_variance==0, which would give nan in the expression below + % If residual variance is 0, want the output to be 1, even if total variance + % is also zero. (The general expression below will be nan if both are zero.) + result = 1 ; else - result = 1 - residual_variance/total_variance ; % this can actually be negative, since the mj2_stack is not a regression + result = 1 - residual_variance/total_variance ; + % this can actually be negative, since the output_stack is not a generally a regression end end else diff --git a/freeze_2018_03_09_rendered.m b/freeze_2018_03_09_rendered.m deleted file mode 100644 index 76cea45..0000000 --- a/freeze_2018_03_09_rendered.m +++ /dev/null @@ -1,7 +0,0 @@ -sample_date = '2018-03-09' ; -mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s-rendered', sample_date) ; -tif_input_folder_path = sprintf('/nrs/mouselight/SAMPLES/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; - -freeze_sample diff --git a/freeze_2018_07_02.m b/freeze_2018_07_02.m deleted file mode 100644 index d432c23..0000000 --- a/freeze_2018_07_02.m +++ /dev/null @@ -1,7 +0,0 @@ -sample_date = '2018-07-02' ; -mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; - -freeze_sample diff --git a/freeze_2018_08_01.m b/freeze_2018_08_01.m deleted file mode 100644 index d127bab..0000000 --- a/freeze_2018_08_01.m +++ /dev/null @@ -1,7 +0,0 @@ -sample_date = '2018-08-01' ; -mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; - -freeze_sample diff --git a/freeze_2018_10_01.m b/freeze_2018_10_01.m deleted file mode 100644 index 49763bd..0000000 --- a/freeze_2018_10_01.m +++ /dev/null @@ -1,7 +0,0 @@ -sample_date = '2018-10-01' ; -mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; - -freeze_sample diff --git a/freeze_2018_12_01.m b/freeze_2018_12_01.m deleted file mode 100644 index a5a7b18..0000000 --- a/freeze_2018_12_01.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2018-12-01' ; -mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -%tif_input_folder_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_1_line_fix_output', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; - -freeze_sample diff --git a/freeze_2019_03_26_raw_tiles.m b/freeze_2019_03_26_raw_tiles.m deleted file mode 100644 index 014433d..0000000 --- a/freeze_2019_03_26_raw_tiles.m +++ /dev/null @@ -1,7 +0,0 @@ -sample_date = '2019-03-26' ; -mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_verify = true ; -do_run_on_cluster = true ; -mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, compression_ratio, do_verify, do_run_on_cluster) diff --git a/freeze_2019_04_17.m b/freeze_2019_04_17.m deleted file mode 100644 index 99abdfa..0000000 --- a/freeze_2019_04_17.m +++ /dev/null @@ -1,7 +0,0 @@ -sample_date = '2019-04-17' ; -mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; - -freeze_sample diff --git a/freeze_2019_05_27.m b/freeze_2019_05_27.m deleted file mode 100644 index 51111fb..0000000 --- a/freeze_2019_05_27.m +++ /dev/null @@ -1,7 +0,0 @@ -sample_date = '2019-05-27' ; -mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; - -freeze_sample diff --git a/freeze_2019_07_19.m b/freeze_2019_07_19.m deleted file mode 100644 index 4faae3c..0000000 --- a/freeze_2019_07_19.m +++ /dev/null @@ -1,7 +0,0 @@ -sample_date = '2019-07-19' ; -mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; - -freeze_sample diff --git a/freeze_2019_08_08.m b/freeze_2019_08_08.m deleted file mode 100644 index b063870..0000000 --- a/freeze_2019_08_08.m +++ /dev/null @@ -1,7 +0,0 @@ -sample_date = '2019-08-08' ; -mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; - -freeze_sample diff --git a/freeze_2019_08_22.m b/freeze_2019_08_22.m deleted file mode 100644 index 8116e86..0000000 --- a/freeze_2019_08_22.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2019-08-22' ; -mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/RAW_archive/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2019_09_06.m b/freeze_2019_09_06.m deleted file mode 100644 index ed61cd5..0000000 --- a/freeze_2019_09_06.m +++ /dev/null @@ -1,7 +0,0 @@ -sample_date = '2019-09-06' ; -mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; - -freeze_sample diff --git a/freeze_2019_10_04.m b/freeze_2019_10_04.m deleted file mode 100644 index 0de121f..0000000 --- a/freeze_2019_10_04.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2019-10-04' ; -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 1500 ; - -freeze_sample diff --git a/freeze_2019_10_31.m b/freeze_2019_10_31.m deleted file mode 100644 index 1871a7f..0000000 --- a/freeze_2019_10_31.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2019-10-31' ; -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 1500 ; - -freeze_sample diff --git a/freeze_2019_12_25.m b/freeze_2019_12_25.m deleted file mode 100644 index 03ffb27..0000000 --- a/freeze_2019_12_25.m +++ /dev/null @@ -1,7 +0,0 @@ -sample_date = '2019-12-25' ; -mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; - -freeze_sample diff --git a/freeze_2020_01_13.m b/freeze_2020_01_13.m deleted file mode 100644 index 04b3a2d..0000000 --- a/freeze_2020_01_13.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2020-01-13' ; -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 1500 ; - -freeze_sample diff --git a/freeze_2020_01_23.m b/freeze_2020_01_23.m deleted file mode 100644 index 4d9816b..0000000 --- a/freeze_2020_01_23.m +++ /dev/null @@ -1,9 +0,0 @@ -sample_date = '2020-01-23' ; -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample - diff --git a/freeze_2020_02_01.m b/freeze_2020_02_01.m deleted file mode 100644 index b613fc2..0000000 --- a/freeze_2020_02_01.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2020-02-01' ; -mj2_output_folder_path = sprintf('/nrs/mouselight/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2020_02_24.m b/freeze_2020_02_24.m deleted file mode 100644 index 68ba191..0000000 --- a/freeze_2020_02_24.m +++ /dev/null @@ -1,9 +0,0 @@ -sample_date = '2020-02-24' ; -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample - diff --git a/freeze_2020_04_15.m b/freeze_2020_04_15.m deleted file mode 100644 index 490d7ab..0000000 --- a/freeze_2020_04_15.m +++ /dev/null @@ -1,9 +0,0 @@ -sample_date = '2020-04-15' ; -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample - diff --git a/freeze_2020_09_15.m b/freeze_2020_09_15.m deleted file mode 100644 index c8c224c..0000000 --- a/freeze_2020_09_15.m +++ /dev/null @@ -1,9 +0,0 @@ -sample_date = '2020-09-15' ; -mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample - diff --git a/freeze_2020_10_16.m b/freeze_2020_10_16.m deleted file mode 100644 index 0e12445..0000000 --- a/freeze_2020_10_16.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2020-10-16' ; -mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/RAW_archive/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2020_10_16_driver_script.sh b/freeze_2020_10_16_driver_script.sh deleted file mode 100644 index f6b4d60..0000000 --- a/freeze_2020_10_16_driver_script.sh +++ /dev/null @@ -1,3 +0,0 @@ -cd /groups/mousebrainmicro/mousebrainmicro/scripts/freezing/ -out=freeze-2022-10-16.out.txt -bsub -P mouselight -n 8 -oo $out -eo $out /misc/local/matlab-2019a/bin/matlab -nodisplay -batch "modpath; freeze_2020_10_16" diff --git a/freeze_2020_11_26.m b/freeze_2020_11_26.m deleted file mode 100644 index 7dc1101..0000000 --- a/freeze_2020_11_26.m +++ /dev/null @@ -1,9 +0,0 @@ -sample_date = '2020-11-26' ; -mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample - diff --git a/freeze_2020_11_26_compression_test.m b/freeze_2020_11_26_compression_test.m deleted file mode 100644 index 907ab0c..0000000 --- a/freeze_2020_11_26_compression_test.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2020-11-26' ; -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/test-data/test-unscaled/compression-tests/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/test-data/test-unscaled/2020-11-26/') ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2020_12_31.m b/freeze_2020_12_31.m deleted file mode 100644 index ba8adff..0000000 --- a/freeze_2020_12_31.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2020-12-31' ; -mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/RAW_archive/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2020_12_31_driver_script.sh b/freeze_2020_12_31_driver_script.sh deleted file mode 100644 index d4d8527..0000000 --- a/freeze_2020_12_31_driver_script.sh +++ /dev/null @@ -1,3 +0,0 @@ -cd /groups/mousebrainmicro/mousebrainmicro/scripts/freezing/ -out=freeze-2022-12-31.out.txt -bsub -P mouselight -n 8 -oo $out -eo $out /misc/local/matlab-2019a/bin/matlab -nodisplay -batch "modpath; freeze_2020_12_31" diff --git a/freeze_2021_02_01.m b/freeze_2021_02_01.m deleted file mode 100644 index 0a2d87b..0000000 --- a/freeze_2021_02_01.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2022-05-10' ; -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2021_02_05.m b/freeze_2021_02_05.m deleted file mode 100644 index 1a14f01..0000000 --- a/freeze_2021_02_05.m +++ /dev/null @@ -1,9 +0,0 @@ -sample_date = '2021-02-05' ; -mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/RAW_archive/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample - diff --git a/freeze_2021_03_17.m b/freeze_2021_03_17.m deleted file mode 100644 index 55c3066..0000000 --- a/freeze_2021_03_17.m +++ /dev/null @@ -1,9 +0,0 @@ -sample_date = '2021-03-17' ; -mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE_TEMP/RAW_archive/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample - diff --git a/freeze_2021_04_10.m b/freeze_2021_04_10.m deleted file mode 100644 index f2d19d8..0000000 --- a/freeze_2021_04_10.m +++ /dev/null @@ -1,9 +0,0 @@ -sample_date = '2021-04-10' ; -mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/RAW_archive/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample - diff --git a/freeze_2021_05_22.m b/freeze_2021_05_22.m deleted file mode 100644 index 4633aad..0000000 --- a/freeze_2021_05_22.m +++ /dev/null @@ -1,9 +0,0 @@ -sample_date = '2021-05-22' ; -mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE_TEMP/RAW_archive/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample - diff --git a/freeze_2021_06_06.m b/freeze_2021_06_06.m deleted file mode 100644 index a10630f..0000000 --- a/freeze_2021_06_06.m +++ /dev/null @@ -1,9 +0,0 @@ -sample_date = '2021-06-06' ; -mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/RAW_archive/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample - diff --git a/freeze_2021_07_20.m b/freeze_2021_07_20.m deleted file mode 100644 index 385332f..0000000 --- a/freeze_2021_07_20.m +++ /dev/null @@ -1,7 +0,0 @@ -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/2021-07-20/') ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_render/2021-07-20/Tiling/') ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2021_07_20_alt.m b/freeze_2021_07_20_alt.m deleted file mode 100644 index 31a22f8..0000000 --- a/freeze_2021_07_20_alt.m +++ /dev/null @@ -1,7 +0,0 @@ -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/2021-07-20') ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_render/2021-07-20/Tiling') ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2021_07_20_debug.m b/freeze_2021_07_20_debug.m deleted file mode 100644 index e98e100..0000000 --- a/freeze_2021_07_20_debug.m +++ /dev/null @@ -1,7 +0,0 @@ -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/2021-07-20/') ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_render/2021-07-20/Tiling/') ; -compression_ratio = 10 ; -do_run_on_cluster = false ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2021_07_20b.m b/freeze_2021_07_20b.m deleted file mode 100644 index 65a0d67..0000000 --- a/freeze_2021_07_20b.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2021-07-20' ; -mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/RAW_archive/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2021_07_20b_driver_script.sh b/freeze_2021_07_20b_driver_script.sh deleted file mode 100644 index ba68f6e..0000000 --- a/freeze_2021_07_20b_driver_script.sh +++ /dev/null @@ -1,3 +0,0 @@ -cd /groups/mousebrainmicro/mousebrainmicro/scripts/freezing/ -out=freeze-2022-07-20b.out.txt -bsub -P mouselight -n 8 -oo $out -eo $out /misc/local/matlab-2019a/bin/matlab -nodisplay -batch "modpath; freeze_2021_07_20b" diff --git a/freeze_2021_08_17.m b/freeze_2021_08_17.m deleted file mode 100644 index 25d75b4..0000000 --- a/freeze_2021_08_17.m +++ /dev/null @@ -1,9 +0,0 @@ -sample_date = '2021-08-17' ; -mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE_TEMP/RAW_archive/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/aborted-acquisitions/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample - diff --git a/freeze_2021_09_16.m b/freeze_2021_09_16.m deleted file mode 100644 index 2dcfdf1..0000000 --- a/freeze_2021_09_16.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2021-09-16' ; -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2021_10_07.m b/freeze_2021_10_07.m deleted file mode 100644 index b38c11b..0000000 --- a/freeze_2021_10_07.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2021-10-07' ; -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2021_10_19.m b/freeze_2021_10_19.m deleted file mode 100644 index cb609e6..0000000 --- a/freeze_2021_10_19.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2021-10-19' ; -mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/RAW_archive/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2021_12_17.m b/freeze_2021_12_17.m deleted file mode 100644 index 26aa1e9..0000000 --- a/freeze_2021_12_17.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2021-12-17' ; -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2022_02_01.m b/freeze_2022_02_01.m deleted file mode 100644 index 182356a..0000000 --- a/freeze_2022_02_01.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2022-02-01' ; -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2022_03_23.m b/freeze_2022_03_23.m deleted file mode 100644 index aeda5e1..0000000 --- a/freeze_2022_03_23.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2022-03-23' ; -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2022_04_25.m b/freeze_2022_04_25.m deleted file mode 100644 index d3358c1..0000000 --- a/freeze_2022_04_25.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2022-04-25' ; -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2022_05_10.m b/freeze_2022_05_10.m deleted file mode 100644 index 0a2d87b..0000000 --- a/freeze_2022_05_10.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2022-05-10' ; -mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/freeze_2022_05_30.m b/freeze_2022_05_30.m index d222e5b..eda30c7 100644 --- a/freeze_2022_05_30.m +++ b/freeze_2022_05_30.m @@ -3,6 +3,15 @@ tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ; compression_ratio = 10 ; do_run_on_cluster = true ; +do_try = true ; maximum_running_slot_count = 500 ; +submit_host_name = if_not_a_submit_host('login2.int.janelia.org') ; -freeze_sample +freeze_mouselight_folder(... + mj2_output_folder_path, ... + tif_input_folder_path, ... + compression_ratio, ... + do_run_on_cluster, ... + do_try, ... + maximum_running_slot_count, ... + submit_host_name) ; diff --git a/freeze_mouselight_folder.m b/freeze_mouselight_folder.m new file mode 100644 index 0000000..f6628ab --- /dev/null +++ b/freeze_mouselight_folder.m @@ -0,0 +1,78 @@ +function freeze_mouselight_folder(... + mj2_output_folder_path, ... + tif_input_folder_path, ... + compression_ratio, ... + do_run_on_cluster, ... + do_try, ... + maximum_running_slot_count, ... + submit_host_name) + + % + % Compress full-rez octree tiffs to .mj2, copy all non-tif files over + % + + % Compress all the full-rez octree tiles to .mj2 + batch_convert_all_tif_files_to_mj2(mj2_output_folder_path, ... + tif_input_folder_path, ... + compression_ratio, ... + do_run_on_cluster, ... + do_try, ... + maximum_running_slot_count, ... + submit_host_name) ; + + % Copy all the non-tiff files over to the tif side + copy_non_tif_files_to_mj2_side(mj2_output_folder_path, ... + tif_input_folder_path) ; + + % + % Verify that all that went well + % + + % Use the cluster to check that all compressed .tif's match their .mj2 well + % enough, and write a 'certificate' file to represent that. + fprintf('Doing batch preverification of .tif files...\n') ; + batch_preverify_tif_files_after_mj2_from_tif(mj2_output_folder_path, ... + tif_input_folder_path, ... + do_run_on_cluster, ... + do_try, ... + submit_host_name, ... + maximum_running_slot_count) + fprintf('Done with batch preverification of .tif files.\n') ; + + % Wait a bit to make sure the filesystem changes have propagated + if do_run_on_cluster , + pause(20) ; + end + + % Check that all the cetificate files that should be there, are there + fprintf('Doing verification of .tif files...\n') ; + verify_tif_files_after_mj2_from_tif(tif_input_folder_path, mj2_output_folder_path) ; + fprintf('Done with verification of .tif files.\n') ; + + % Verify that all the non-tif files exist on the mj2 side + fprintf('Doing verification of non-.tif files...\n') ; + verify_non_tif_files_after_mj2_from_tif(mj2_output_folder_path, tif_input_folder_path) ; + fprintf('Done with verification of non-.tif files.\n') ; + + % + % Do a final comparison of .tif files and .mj2 files, just for the heck of it + % + + % Count the tifs + final_tif_count = ... + find_and_count(tif_input_folder_path, @does_file_name_end_in_dot_tif) ; + fprintf('Final .tif count in %s is: %d\n', tif_input_folder_path, final_tif_count) ; + + % Count the mj2s + final_mj2_count = ... + find_and_count(mj2_output_folder_path, @does_file_name_end_in_dot_mj2) ; + fprintf('Final .mj2 count in %s is: %d\n', mj2_output_folder_path, final_mj2_count) ; + + % Final check + if final_tif_count ~= final_mj2_count , + error('Final .mj2 count in target folder differs from final .tif count in source folder') ; + end + + % Report final status if we get this far + fprintf('Octree compression and verification succeeded!\n') ; +end diff --git a/freeze_nrs_test_tf.m b/freeze_nrs_test_tf.m deleted file mode 100644 index 868907c..0000000 --- a/freeze_nrs_test_tf.m +++ /dev/null @@ -1,9 +0,0 @@ -sample_date = '2020-11-30-selected' ; -mj2_output_folder_path = sprintf('/nrs/mouselight/Temp/frozen-test/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/acquisition/scope1-tests/%s', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample - diff --git a/freeze_octree_2023_02_23.m b/freeze_octree_2023_02_23.m new file mode 100644 index 0000000..d0e4708 --- /dev/null +++ b/freeze_octree_2023_02_23.m @@ -0,0 +1,17 @@ +sample_date = '2023-02-23' ; +mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/2023-02-23/%s', sample_date) ; +tif_input_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/%s', sample_date) ; +compression_ratio = 10 ; +do_run_on_cluster = true ; +do_try = true ; % not used +maximum_running_slot_count = 500 ; +submit_host_name = if_not_a_submit_host('login2.int.janelia.org') ; + +freeze_mouselight_folder(mj2_output_folder_path, ... + tif_input_folder_path, ... + compression_ratio, ... + do_run_on_cluster, ... + do_try, ... + maximum_running_slot_count, ... + submit_host_name) ; + diff --git a/freeze_sample.m b/freeze_sample.m deleted file mode 100644 index d826daa..0000000 --- a/freeze_sample.m +++ /dev/null @@ -1,43 +0,0 @@ -% Actually do the compression, by launching a bunch of batch jobs -mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, compression_ratio, do_run_on_cluster, maximum_running_slot_count) - -fprintf('Doing batch preverification of .tif files...\n') ; -batch_preverify_tif_files_after_mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, do_run_on_cluster, maximum_running_slot_count) -fprintf('Done with batch preverification of .tif files.\n') ; - -fprintf('Doing verification of .tif files...\n') ; -verify_tif_files_after_mj2_from_tif(tif_input_folder_path, mj2_output_folder_path) ; -fprintf('Done with verification of .tif files.\n') ; - -fprintf('Doing verification of non-.tif files...\n') ; -verify_non_tif_files_after_mj2_from_tif(mj2_output_folder_path, tif_input_folder_path) ; -fprintf('Done with verification of non-.tif files.\n') ; - -% Decided this is a bad idea because it breaks idemptoency, at least -% in a practical sense. You don't ever want to re-verify files that you've -% verified. Just leave the certificates in place until there's a problem. -% -% % Delete certificate files (just run locally) -% find_and_batch(mj2_output_folder_path, ... -% @does_file_name_match_certificate_file, ... -% @delete, ... -% false, ... -% '') ; - -% Do a final comparison of .tif files and .mj2 files, just for the heck of it -[status, result] = system(sprintf('find %s -name "*.tif" -print | wc -l', tif_input_folder_path)) ; -if status~= 0 , - error('Unable to count number of .tif files\n') ; -end -final_tif_count = str2double(result) ; -fprintf('.tif count in %s is: %d\n', tif_input_folder_path, final_tif_count) ; - -[status, result] = system(sprintf('find %s -name "*.mj2" -print | wc -l', mj2_output_folder_path)) ; -if status~= 0 , - error('Unable to count final number of .mj2 files\n') ; -end -final_mj2_count = str2double(result) ; -fprintf('Final .mj2 count in %s is: %d\n', mj2_output_folder_path, final_mj2_count) ; - -% Report final status if we get this far -fprintf('Compression and verification succeeded!\n') ; diff --git a/freeze_temp.m b/freeze_temp.m deleted file mode 100644 index 6f4808d..0000000 --- a/freeze_temp.m +++ /dev/null @@ -1,8 +0,0 @@ -sample_date = '2020-04-15' ; -mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/RAW_archive/%s', sample_date) ; -tif_input_folder_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_1_line_fix_output/', sample_date) ; -compression_ratio = 10 ; -do_run_on_cluster = true ; -maximum_running_slot_count = 500 ; - -freeze_sample diff --git a/is_a_mj2_and_target_missing.m b/is_a_mj2_and_target_missing.m new file mode 100644 index 0000000..8a1c99e --- /dev/null +++ b/is_a_mj2_and_target_missing.m @@ -0,0 +1,12 @@ +function result = is_a_mj2_and_target_missing(input_file_path, input_root_path, output_root_path, varargin) + is_true_so_far = does_file_name_end_in_dot_mj2(input_file_path) ; + if ~is_true_so_far , + result = false ; + return + end + % Check to see if the target is missing + relative_file_path_of_input = relpath(input_file_path, input_root_path) ; + relative_file_path_of_output= replace_extension(relative_file_path_of_input, '.tif') ; + output_file_path = fullfile(output_root_path, relative_file_path_of_output) ; + result = ~logical(exist(output_file_path, 'file')) ; +end diff --git a/is_a_mj2_in_a_folder_with_no_subfolders.m b/is_a_mj2_in_a_folder_with_no_subfolders.m new file mode 100644 index 0000000..5e7c3e0 --- /dev/null +++ b/is_a_mj2_in_a_folder_with_no_subfolders.m @@ -0,0 +1,9 @@ +function result = is_a_mj2_in_a_folder_with_no_subfolders(input_file_path, varargin) + [input_folder_path, input_file_name] = fileparts2(input_file_path) ; + is_true_so_far = does_file_name_end_in_dot_mj2(input_file_name) ; + if ~is_true_so_far , + result = false ; + return + end + result = ~does_folder_have_subfolders(input_folder_path) ; +end diff --git a/is_a_mj2_in_a_folder_with_no_subfolders_and_target_missing.m b/is_a_mj2_in_a_folder_with_no_subfolders_and_target_missing.m new file mode 100644 index 0000000..d51b4d5 --- /dev/null +++ b/is_a_mj2_in_a_folder_with_no_subfolders_and_target_missing.m @@ -0,0 +1,12 @@ +function result = is_a_mj2_in_a_folder_with_no_subfolders_and_target_missing(input_file_path, input_root_path, output_root_path, varargin) + is_true_so_far = is_a_mj2_in_a_folder_with_no_subfolders(input_file_path) ; + if ~is_true_so_far , + result = false ; + return + end + % Check to see if the target is missing + relative_file_path_of_input = relpath(input_file_path, input_root_path) ; + relative_file_path_of_output= replace_extension(relative_file_path_of_input, '.tif') ; + output_file_path = fullfile(output_root_path, relative_file_path_of_output) ; + result = ~logical(exist(output_file_path, 'file')) ; +end diff --git a/is_a_non_mj2_and_target_missing.m b/is_a_non_mj2_and_target_missing.m new file mode 100644 index 0000000..15729d1 --- /dev/null +++ b/is_a_non_mj2_and_target_missing.m @@ -0,0 +1,13 @@ +function result = is_a_non_mj2_and_target_missing(input_file_path, input_root_path, output_root_path, varargin) + [~, input_file_name] = fileparts2(input_file_path) ; + is_true_so_far = ~does_file_name_end_in_dot_mj2(input_file_name) ; + if ~is_true_so_far , + result = false ; + return + end + % Check to see if the target is missing + relative_file_path_of_input_file = relpath(input_file_path, input_root_path) ; + relative_file_path_of_output_file = relative_file_path_of_input_file ; + output_file_path = fullfile(output_root_path, relative_file_path_of_output_file) ; + result = ~exist(output_file_path, 'file') ; +end diff --git a/is_a_non_tif_and_target_missing.m b/is_a_non_tif_and_target_missing.m new file mode 100644 index 0000000..23f0107 --- /dev/null +++ b/is_a_non_tif_and_target_missing.m @@ -0,0 +1,13 @@ +function result = is_a_non_tif_and_target_missing(tif_side_file_path, tif_root_path, mj2_root_path, varargin) + [~, file_name] = fileparts2(tif_side_file_path) ; + is_true_so_far = ~does_file_name_end_in_dot_tif(file_name) ; + if ~is_true_so_far , + result = false ; + return + end + % Check to see if the target is missing + relative_file_path_of_tif_side_file = relpath(tif_side_file_path, tif_root_path) ; + relative_file_path_of_mj2_side_file = relative_file_path_of_tif_side_file ; + mj2_side_file_path = fullfile(mj2_root_path, relative_file_path_of_mj2_side_file) ; + result = ~exist(mj2_side_file_path, 'file') ; +end diff --git a/is_a_tif_and_target_present.m b/is_a_tif_and_target_present.m new file mode 100644 index 0000000..a278af3 --- /dev/null +++ b/is_a_tif_and_target_present.m @@ -0,0 +1,12 @@ +function result = is_a_tif_and_target_present(tif_file_path, tif_root_path, mj2_root_path, varargin) + is_true_so_far = does_file_name_end_in_dot_tif(tif_file_path) ; + if ~is_true_so_far , + result = false ; + return + end + % Check to see if the target is present + relative_file_path_of_tif = relpath(tif_file_path, tif_root_path) ; + relative_file_path_of_mj2 = replace_extension(relative_file_path_of_tif, '.mj2') ; + mj2_file_path = fullfile(mj2_root_path, relative_file_path_of_mj2) ; + result = logical(exist(mj2_file_path, 'file')) ; +end diff --git a/is_a_tif_in_a_folder_with_no_subfolders.m b/is_a_tif_in_a_folder_with_no_subfolders.m new file mode 100644 index 0000000..8e8d8c9 --- /dev/null +++ b/is_a_tif_in_a_folder_with_no_subfolders.m @@ -0,0 +1,9 @@ +function result = is_a_tif_in_a_folder_with_no_subfolders(tif_file_path, varargin) + [tif_folder_path,file_name] = fileparts2(tif_file_path) ; + is_true_so_far = does_file_name_end_in_dot_tif(file_name) ; + if ~is_true_so_far , + result = false ; + return + end + result = ~does_folder_have_subfolders(tif_folder_path) ; +end diff --git a/is_a_tif_in_a_folder_with_no_subfolders_and_target_missing.m b/is_a_tif_in_a_folder_with_no_subfolders_and_target_missing.m new file mode 100644 index 0000000..2aefdd6 --- /dev/null +++ b/is_a_tif_in_a_folder_with_no_subfolders_and_target_missing.m @@ -0,0 +1,12 @@ +function result = is_a_tif_in_a_folder_with_no_subfolders_and_target_missing(tif_file_path, tif_root_path, mj2_root_path, varargin) + is_true_so_far = is_a_tif_in_a_folder_with_no_subfolders(tif_file_path) ; + if ~is_true_so_far , + result = false ; + return + end + % Check to see if the target is missing + relative_file_path_of_tif = relpath(tif_file_path, tif_root_path) ; + relative_file_path_of_mj2 = replace_extension(relative_file_path_of_tif, '.mj2') ; + mj2_file_path = fullfile(mj2_root_path, relative_file_path_of_mj2) ; + result = ~logical(exist(mj2_file_path, 'file')) ; +end diff --git a/is_a_tif_in_a_folder_with_no_subfolders_and_target_present.m b/is_a_tif_in_a_folder_with_no_subfolders_and_target_present.m new file mode 100644 index 0000000..0fe81dd --- /dev/null +++ b/is_a_tif_in_a_folder_with_no_subfolders_and_target_present.m @@ -0,0 +1,12 @@ +function result = is_a_tif_in_a_folder_with_no_subfolders_and_target_present(tif_file_path, tif_root_path, mj2_root_path, varargin) + is_true_so_far = is_a_tif_in_a_folder_with_no_subfolders(tif_file_path) ; + if ~is_true_so_far , + result = false ; + return + end + % Check to see if the target is present + relative_file_path_of_tif = relpath(tif_file_path, tif_root_path) ; + relative_file_path_of_mj2 = replace_extension(relative_file_path_of_tif, '.mj2') ; + mj2_file_path = fullfile(mj2_root_path, relative_file_path_of_mj2) ; + result = logical(exist(mj2_file_path, 'file')) ; +end diff --git a/is_file_below_depth_limit.m b/is_file_below_depth_limit.m new file mode 100644 index 0000000..73b56ae --- /dev/null +++ b/is_file_below_depth_limit.m @@ -0,0 +1,10 @@ +function result = is_file_below_depth_limit(file_path, root_folder_path, depth_limit, varargin) + % Files in the root folder are at depth zero, files in subdirs at depth one, + % etc. Returns true for files at depth less than or equal to depth_limit. + relative_file_path = relpath(file_path, root_folder_path) ; + [relative_folder_path, ~] = fileparts2(relative_file_path) ; + relative_folder_path_as_object = path_object(relative_folder_path) ; + relative_folder_path_as_list = relative_folder_path_as_object.list() ; + depth = length(relative_folder_path_as_list) ; + result = (depth<=depth_limit) ; +end diff --git a/is_leaf_mj2_and_lacks_cert_for_output.m b/is_leaf_mj2_and_lacks_cert_for_output.m new file mode 100644 index 0000000..eb4f321 --- /dev/null +++ b/is_leaf_mj2_and_lacks_cert_for_output.m @@ -0,0 +1,18 @@ +function result = is_leaf_mj2_and_lacks_cert_for_output(input_file_path, input_root_path, output_root_path) + is_input_file_an_input_leaf_image = is_a_mj2_in_a_folder_with_no_subfolders(input_file_path) ; + if ~is_input_file_an_input_leaf_image , + result = false ; + return + end +% % Check to see if the output is present +% relative_input_file_path = relpath(input_file_path, input_root_path) ; +% relative_output_file_path = replace_extension(relative_input_file_path, '.tif') ; +% output_file_path = fullfile(output_root_path, relative_output_file_path) ; +% does_output_file_exist = logical(exist(output_file_path, 'file')) ; +% if ~does_output_file_exist , +% result = true ; +% return +% end + does_cert_exist = does_is_similar_to_tif_check_file_exist(input_file_path, input_root_path, output_root_path) ; + result = ~does_cert_exist ; +end diff --git a/is_leaf_mj2_and_lacks_either_output_or_cert.m b/is_leaf_mj2_and_lacks_either_output_or_cert.m new file mode 100644 index 0000000..84e3731 --- /dev/null +++ b/is_leaf_mj2_and_lacks_either_output_or_cert.m @@ -0,0 +1,20 @@ +function result = is_leaf_mj2_and_lacks_either_output_or_cert(input_file_path, input_root_path, output_root_path) + is_input_file_an_input_leaf_image = is_a_mj2_in_a_folder_with_no_subfolders(input_file_path) ; + if ~is_input_file_an_input_leaf_image , + result = false ; + return + end + % Check to see if the output is present + relative_input_file_path = relpath(input_file_path, input_root_path) ; + relative_output_file_path = replace_extension(relative_input_file_path, '.tif') ; + output_file_path = fullfile(output_root_path, relative_output_file_path) ; + does_output_file_exist = logical(exist(output_file_path, 'file')) ; + if ~does_output_file_exist , + result = true ; + return + end + % If get here, input file is a leaf image, and the output file is present + % Check for certificate + does_cert_exist = does_is_similar_to_tif_check_file_exist(input_file_path, input_root_path, output_root_path) ; + result = +end diff --git a/is_mj2_same_as_tif.m b/is_mj2_same_as_tif.m deleted file mode 100644 index 04938a9..0000000 --- a/is_mj2_same_as_tif.m +++ /dev/null @@ -1,9 +0,0 @@ -function result = is_mj2_same_as_tif(mj2_file_name, tif_file_name) - % Converts a single .mj2 file at input_file_name to a multi-image .tif - % at output_file_name. Will overwrite pre-existing file at - % output_file_name, if present. - - mj2_stack = read_16bit_grayscale_mj2(mj2_file_name) ; - tif_stack = read_16bit_grayscale_tif(tif_file_name) ; - result = isequal(class(mj2_stack), class(tif_stack)) && isequal(size(mj2_stack), size(tif_stack)) && all(all(all(mj2_stack==tif_stack))) ; -end diff --git a/is_mj2_similar_to_tif.m b/is_mj2_similar_to_tif.m deleted file mode 100644 index 7770d0d..0000000 --- a/is_mj2_similar_to_tif.m +++ /dev/null @@ -1,23 +0,0 @@ -function result = is_mj2_similar_to_tif(mj2_stack, tif_stack) - % Checks if an mj2 file is similar to a .tif file. - % Although really, can compare any two stacks. - - %mj2_stack = read_16bit_grayscale_mj2(mj2_file_name) ; - %tif_stack = read_16bit_grayscale_tif(tif_file_name) ; - if isequal(class(mj2_stack), class(tif_stack)) && isequal(size(mj2_stack), size(tif_stack)) , - if all(all(all(mj2_stack==tif_stack))) , - result = true ; - else - % ssim_value = ssim(mj2_stack, tif_stack) ; - % result = (ssim_value>0.95) ; - - % mean_summary_error = voxelwise_mean_relative_error_with_guard(mj2_stack, tif_stack, 1/256) ; - % result = (mean_summary_error<0.1) ; % this is just intended as a gross sanity check, so use a tolerant threshold - - fpe_value = fraction_variance_explained(mj2_stack, tif_stack) ; - result = (fpe_value > 0.75) ; - end - else - result = false ; - end -end diff --git a/is_mj2_similar_to_tif_from_paths.m b/is_mj2_similar_to_tif_from_paths.m deleted file mode 100644 index 90329bb..0000000 --- a/is_mj2_similar_to_tif_from_paths.m +++ /dev/null @@ -1,23 +0,0 @@ -function are_files_the_same = is_mj2_similar_to_tif_from_paths(mj2_file_path, tif_file_path) - if exist(mj2_file_path, 'file') , - try - mj2_stack = read_16bit_grayscale_mj2(mj2_file_path) ; - catch err - are_files_the_same = false ; - fprintf('Unable to read target file %s: %s\n', mj2_file_path, err.message) ; - return - end - tif_stack = read_16bit_grayscale_tif(tif_file_path) ; - if is_mj2_similar_to_tif(mj2_stack, tif_stack) , - are_files_the_same = true ; - else - are_files_the_same = false ; - fprintf('Target file %s is considerably different from source file%s\n', ... - mj2_file_path, ... - tif_file_path) ; - end - else - are_files_the_same = false ; - fprintf('Target file %s is missing\n', mj2_file_path) ; - end -end diff --git a/mj2_from_tif.m b/mj2_from_tif.m deleted file mode 100644 index f04d019..0000000 --- a/mj2_from_tif.m +++ /dev/null @@ -1,92 +0,0 @@ -function mj2_from_tif(mj2_root_folder_path, tif_root_folder_path, compression_ratio, do_run_on_cluster, maximum_running_slot_count) - % Converts each .mj2 file in input_folder_name to a multi-image .tif in - % output_folder_name. Will overwrite pre-existing files in - % output_folder_name, if present. - - if ~exist('compression_ratio', 'var') || isempty(compression_ratio) , - compression_ratio = 10 ; - end - if ~exist('do_run_on_cluster', 'var') || isempty(do_run_on_cluster) , - do_run_on_cluster = false ; - end - if ~exist('maximum_running_slot_count', 'var') || isempty(maximum_running_slot_count) , - maximum_running_slot_count = inf ; - end - - % Call the helper - fprintf('Compressing .tifs to .mj2s...\n') ; - slots_per_job = 1 ; - bsub_options = '-P mouselight -W 59 -J freeze' ; - bqueue = bqueue_type(do_run_on_cluster, maximum_running_slot_count) ; - mj2_from_tif_helper_bang(bqueue, bsub_options, slots_per_job, mj2_root_folder_path, tif_root_folder_path, compression_ratio) ; - - % Wait for the jobs to finish - fprintf('Waiting for %d mj2_from_tif() bjobs to finish...\n', bqueue.queue_length()) ; - %bwait(job_ids) ; - bqueue.run() ; - fprintf('mj2_from_tif() bjobs are done.\n') ; - -% bsub_options = '-n1 -P mouselight -W 59 -J freeze' ; -% find_and_batch(tif_root_folder_path, ... -% @(varargin)(true), ... -% @mj2_from_tif_single, ... -% do_run_on_cluster, ... -% bsub_options, ... -% tif_root_folder_path, ... -% mj2_root_folder_path, ... -% compression_ratio) ; - - fprintf('Done compressing .tifs to .mj2s.\n') ; -end - - - -function mj2_from_tif_helper_bang(bqueue, bsub_options, slots_per_job, mj2_folder_path, tif_folder_path, compression_ratio) - if ~exist(mj2_folder_path, 'dir') , - mkdir(mj2_folder_path) ; - end - tif_side_file_names = simple_dir(tif_folder_path) ; - tif_side_file_name_count = length(tif_side_file_names) ; - for i = 1 : tif_side_file_name_count , - tif_side_file_name = tif_side_file_names{i} ; - tif_side_file_path = fullfile(tif_folder_path, tif_side_file_name) ; - if exist(tif_side_file_path, 'dir') , - % if a folder, recurse - tif_side_subfolder_path = tif_side_file_path ; - tif_side_subfolder_name = tif_side_file_name ; - mj2_side_subfolder_path = fullfile(mj2_folder_path, tif_side_subfolder_name) ; - mj2_from_tif_helper_bang(bqueue, bsub_options, slots_per_job, mj2_side_subfolder_path, tif_side_subfolder_path, compression_ratio) ; - else - % if a normal file, convert to .mj2 if it's a .tif, or just - % copy otherwise - if does_file_name_end_in_dot_tif(tif_side_file_path) , - tif_file_path = tif_side_file_path ; - tif_file_name = tif_side_file_name ; - mj2_file_name = replace_extension(tif_file_name, '.mj2') ; - mj2_file_path = fullfile(mj2_folder_path, mj2_file_name) ; - if ~exist(mj2_file_path, 'file') , -% job_id = bsub(do_run_on_cluster, ... -% bsub_options, ... -% @mj2_from_tif_single, ... -% mj2_file_path, ... -% tif_file_path, ... -% compression_ratio) ; - fprintf('Enqueuing job for %s\n', tif_file_path) ; - bqueue.enqueue(slots_per_job, ... - [], ... - bsub_options, ... - @mj2_from_tif_single, ... - mj2_file_path, ... - tif_file_path, ... - compression_ratio) ; - %job_ids = horzcat(job_ids, job_id) ; %#ok - end - else - mj2_side_file_path = fullfile(mj2_folder_path, tif_side_file_name) ; % input not really a tif, output not really a .mj2 - if ~exist(mj2_side_file_path, 'file') , - copyfile(tif_side_file_path, mj2_side_file_path) ; - end - end - end - end -end diff --git a/mj2_from_tif_single.m b/mj2_from_tif_single.m index 2b3aa3a..a173fac 100644 --- a/mj2_from_tif_single.m +++ b/mj2_from_tif_single.m @@ -4,4 +4,3 @@ function mj2_from_tif_single(mj2_file_path, tif_file_path, compression_ratio) % Write the output file write_16bit_grayscale_mj2(mj2_file_path, tif_stack, compression_ratio) ; end - \ No newline at end of file diff --git a/mj2_from_tif_single_for_find_and_batch.m b/mj2_from_tif_single_for_find_and_batch.m new file mode 100644 index 0000000..3e6c498 --- /dev/null +++ b/mj2_from_tif_single_for_find_and_batch.m @@ -0,0 +1,9 @@ +function mj2_from_tif_single_for_find_and_batch(tif_file_path, tif_root_folder_name, mj2_root_folder_name, compression_ratio) + % Compress a single .tif to .mj2 + relative_file_path_of_tif = relpath(tif_file_path, tif_root_folder_name) ; + relative_file_path_of_mj2 = replace_extension(relative_file_path_of_tif, '.mj2') ; + mj2_file_path = fullfile(mj2_root_folder_name, relative_file_path_of_mj2) ; + mj2_folder_path = fileparts2(mj2_file_path) ; + ensure_folder_exists(mj2_folder_path) ; + mj2_from_tif_single(mj2_file_path, tif_file_path, compression_ratio) ; +end diff --git a/mj2_from_tif_single_with_temps.m b/mj2_from_tif_single_with_temps.m deleted file mode 100644 index 06f9bb6..0000000 --- a/mj2_from_tif_single_with_temps.m +++ /dev/null @@ -1,10 +0,0 @@ -function mj2_from_tif_single_with_temps(mj2_output_file_path, tif_input_file_path, compression_ratio, do_verify) - scratch_folder_path = get_scratch_folder_path() ; - temporary_tif_input_file_path = [tempname(scratch_folder_path) '.tif'] ; - temporary_mj2_output_file_path = [tempname(scratch_folder_path) '.mj2'] ; % risky, but VideoWriter is dumb and always adds this anyway - copyfile(tif_input_file_path, temporary_tif_input_file_path); - mj2_from_tif_single(temporary_mj2_output_file_path, temporary_tif_input_file_path, compression_ratio, do_verify); - copyfile(temporary_mj2_output_file_path, mj2_output_file_path); - delete(temporary_tif_input_file_path); - delete(temporary_mj2_output_file_path); -end diff --git a/modpath.m b/modpath.m index 8587a21..5487c50 100644 --- a/modpath.m +++ b/modpath.m @@ -1,5 +1,5 @@ function modpath() this_folder_path = fileparts(mfilename('fullpath')) ; - tmt_modpath_path = fullfile(this_folder_path, 'tmt', 'modpath.m') ; - run(tmt_modpath_path) ; + toolbox_modpath_path = fullfile(this_folder_path, 'mouselight_toolbox', 'modpath.m') ; + run(toolbox_modpath_path) ; end diff --git a/mouselight_toolbox b/mouselight_toolbox new file mode 160000 index 0000000..142f553 --- /dev/null +++ b/mouselight_toolbox @@ -0,0 +1 @@ +Subproject commit 142f553986e7846988cb25d6f5603767182cff9b diff --git a/preverify_single_tif_file_after_mj2_from_tif.m b/preverify_single_tif_file_after_mj2_from_tif.m index 32fcb1b..aa53d1f 100644 --- a/preverify_single_tif_file_after_mj2_from_tif.m +++ b/preverify_single_tif_file_after_mj2_from_tif.m @@ -1,6 +1,6 @@ function preverify_single_tif_file_after_mj2_from_tif(tif_file_path, tif_root_folder_name, mj2_root_folder_name) % Checks that the mj2 is similar to the .tif, and outputs a .similar-mj2-exists file in the - % tif folder if the are similar. If unable to verify, writes nothing. + % tif folder if they are similar. If unable to verify, writes nothing. relative_file_path_of_tif = relpath(tif_file_path, tif_root_folder_name) ; relative_file_path_of_mj2 = replace_extension(relative_file_path_of_tif, '.mj2') ; mj2_file_path = fullfile(mj2_root_folder_name, relative_file_path_of_mj2) ; diff --git a/quickhydrate_2018_03_09.m b/quickhydrate_2018_03_09.m deleted file mode 100644 index 5ac3f75..0000000 --- a/quickhydrate_2018_03_09.m +++ /dev/null @@ -1,5 +0,0 @@ -do_verify = true ; -do_on_cluster = false ; -addpath('/groups/mousebrainmicro/mousebrainmicro/scripts/freezing/') -modpath ; -tif_from_mj2('/groups/scicompsoft/home/schauderd/mj2','/groups/scicompsoft/home/schauderd/mj2', do_verify, do_on_cluster) ; \ No newline at end of file diff --git a/rebuild_tif_octree_from_leaves.m b/rebuild_tif_octree_from_leaves.m new file mode 100644 index 0000000..cb4e333 --- /dev/null +++ b/rebuild_tif_octree_from_leaves.m @@ -0,0 +1,5 @@ +function rebuild_tif_octree_from_leaves(root_folder_path) + path_from_leaf_stack_index = ... + find_and_list(root_folder_path, ... + @is_a_tif_in_a_folder_with_no_subfolders) ; +end diff --git a/test_everything.m b/test_everything.m new file mode 100644 index 0000000..24e207c --- /dev/null +++ b/test_everything.m @@ -0,0 +1,9 @@ +function test_everything() + for freezing_or_thawing = { 'freezing', 'thawing' } , + for raw_tiles_or_octree = { 'raw-tiles', 'octree' } , + for local_or_cluster = { 'local', 'cluster' } , + test_something(freezing_or_thawing{1}, raw_tiles_or_octree{1}, local_or_cluster{1}) + end + end + end +end diff --git a/test_freezing_locally.m b/test_freezing_locally.m deleted file mode 100644 index ff1c54f..0000000 --- a/test_freezing_locally.m +++ /dev/null @@ -1,13 +0,0 @@ -path_to_this_folder = fileparts(mfilename('fullpath')) ; -mj2_output_folder_path = fullfile(path_to_this_folder, 'freezing-test-output-folder') ; -tif_input_folder_path = fullfile(path_to_this_folder, 'freezing-test-input-folder') ; -compression_ratio = 10 ; -do_verify = false ; -do_run_on_cluster = false ; -maximum_running_slot_count = 40 ; - -% Delete the test output folder -reset_for_test(mj2_output_folder_path) ; - -% Call the script -freeze_sample diff --git a/test_freezing_on_cluster.m b/test_freezing_on_cluster.m deleted file mode 100644 index 8955266..0000000 --- a/test_freezing_on_cluster.m +++ /dev/null @@ -1,14 +0,0 @@ -path_to_this_script = mfilename('fullpath') ; -path_to_this_folder = fileparts(path_to_this_script) ; -mj2_output_folder_path = fullfile(path_to_this_folder, 'freezing-test-output-folder') ; -tif_input_folder_path = fullfile(path_to_this_folder, 'freezing-test-input-folder') ; -compression_ratio = 10 ; -do_verify = false ; -do_run_on_cluster = true ; -maximum_running_slot_count = 40 ; - -% Delete the test output folder -reset_for_test(mj2_output_folder_path) ; - -% Call the script -freeze_sample diff --git a/test_something.m b/test_something.m new file mode 100644 index 0000000..8bd6e7d --- /dev/null +++ b/test_something.m @@ -0,0 +1,49 @@ +function test_something(freezing_or_thawing, raw_tiles_or_octree, local_or_cluster) + % Test, in one of 2^3==8 possible configurations. + + % Deal with the arguments + if ~strcmp(freezing_or_thawing, 'freezing') && ~strcmp(freezing_or_thawing, 'thawing') , + error('freezing_or_thawing must be ''freezing'' or ''thawing''') ; + end + if ~strcmp(raw_tiles_or_octree, 'raw-tiles') && ~strcmp(raw_tiles_or_octree, 'octree') , + error('raw_tiles_or_octree must be ''raw-tiles'' or ''octree''') ; + end + if ~strcmp(local_or_cluster, 'local') && ~strcmp(local_or_cluster, 'cluster') , + error('local_or_cluster must be ''local'' or ''cluster''') ; + end + + % Define all the needed inputs + path_to_this_folder = fileparts(mfilename('fullpath')) ; + output_folder_name = sprintf('%s-%s-test-output-folder', raw_tiles_or_octree, freezing_or_thawing) ; + output_folder_path = fullfile(path_to_this_folder, output_folder_name) ; + input_folder_name = sprintf('%s-%s-test-input-folder', raw_tiles_or_octree, freezing_or_thawing) ; + input_folder_path = fullfile(path_to_this_folder, input_folder_name) ; + compression_ratio = 10 ; % only used for freezing + do_run_on_cluster = strcmp(local_or_cluster, 'cluster') ; + do_try = false ; % if running locally, don't wrap in try so easier to debug + maximum_running_slot_count = 40 ; + submit_host_name = if_not_a_submit_host('login2.int.janelia.org') ; + + % Delete the test output folder + reset_for_test(output_folder_path) ; + + % Call the script + if strcmp(freezing_or_thawing, 'freezing') , + freeze_mouselight_folder(... + output_folder_path, ... + input_folder_path, ... + compression_ratio, ... + do_run_on_cluster, ... + do_try, ... + maximum_running_slot_count, ... + submit_host_name) ; + else + thaw_mouselight_folder(... + output_folder_path, ... + input_folder_path, ... + do_run_on_cluster, ... + do_try, ... + maximum_running_slot_count, ... + submit_host_name) ; + end +end diff --git a/thaw_2017_10_31_sample.m b/thaw_2017_10_31_sample.m deleted file mode 100644 index c7cb86d..0000000 --- a/thaw_2017_10_31_sample.m +++ /dev/null @@ -1,3 +0,0 @@ -do_verify = false ; -do_on_cluster = true ; -tif_from_mj2('/groups/mousebrainmicro/mousebrainmicro/data/2017-10-31-thawed', '/nrs/mouselight/SAMPLES/compressed_recovered/2017-10-31', do_verify, do_on_cluster) ; diff --git a/thaw_2017_12_19_sample.m b/thaw_2017_12_19_sample.m index bf2d831..9e1b326 100644 --- a/thaw_2017_12_19_sample.m +++ b/thaw_2017_12_19_sample.m @@ -1,3 +1,7 @@ do_verify = false ; do_on_cluster = true ; -tif_from_mj2('/groups/mousebrainmicro/mousebrainmicro/data/2017-12-19-thawed', '/groups/mousebrainmicro/mousebrainmicro/data/2017-12-19', do_verify, do_on_cluster) ; + +tif_from_mj2('/groups/mousebrainmicro/mousebrainmicro/data/2017-12-19-thawed', ... + '/groups/mousebrainmicro/mousebrainmicro/data/2017-12-19', ... + do_verify, ... + do_on_cluster) ; diff --git a/thaw_mouselight_folder.m b/thaw_mouselight_folder.m new file mode 100644 index 0000000..8a3ea41 --- /dev/null +++ b/thaw_mouselight_folder.m @@ -0,0 +1,80 @@ +function thaw_mouselight_folder(... + output_folder_path, ... + input_folder_path, ... + do_run_on_cluster, ... + do_try, ... + maximum_running_slot_count, ... + submit_host_name) + + % + % Phase 1: Decompress all full-rez octree .mj2's to .tif + % + + % Compress all the full-rez octree tiles to .mj2 + batch_convert_all_mj2_files_to_tif(... + output_folder_path, ... + input_folder_path, ... + do_run_on_cluster, ... + do_try, ... + maximum_running_slot_count, ... + submit_host_name) ; + + % Use the cluster to check that all decompressed .tif's match their .mj2 well + % enough, and write a 'certificate' file to represent that. + fprintf('Doing batch verification of leaf .tif files...\n') ; + batch_verify_leaf_tif_files_after_thawing(... + output_folder_path, ... + input_folder_path, ... + do_run_on_cluster, ... + do_try, ... + submit_host_name, ... + maximum_running_slot_count) + fprintf('Done with batch verification of leaf .tif files.\n') ; + + % Wait a bit to make sure the filesystem changes have propagated + if do_run_on_cluster , + pause(20) ; + end + + % Check that all the cetificate files that should be there, are there + fprintf('Checking leaf verification certificates...\n') ; + check_certs_after_verifying_thawing(input_folder_path, output_folder_path) ; + fprintf('Done checking leaf verification certificates.\n') ; + + + % + % Phase 2: Copy all the non-mj2 files over + % + + % Copy all the non-mj2 files over to the tif side + copy_non_mj2_files_to_tif_side(output_folder_path, ... + input_folder_path) ; + + % Verify that all the non-tif files exist on the mj2 side + fprintf('Doing verification of non-.tif files...\n') ; + verify_non_tif_files_after_thawing(output_folder_path, input_folder_path) ; + fprintf('Done with verification of non-.tif files.\n') ; + + + % + % Do a final comparison of leaf .tif files and .mj2 files, just for the heck of it + % + + % Count the mj2s + final_mj2_count = ... + find_and_count(input_folder_path, @does_file_name_end_in_dot_mj2) ; + fprintf('Final .mj2 count in %s is: %d\n', input_folder_path, final_mj2_count) ; + + % Count the tifs + final_tif_count = ... + find_and_count(output_folder_path, @does_file_name_end_in_dot_tif) ; + fprintf('Final .tif count in %s is: %d\n', output_folder_path, final_tif_count) ; + + % Final check + if final_tif_count ~= final_mj2_count , + error('Final .tif count in target folder differs from final .mj2 count in source folder') ; + end + + % Report final status if we get this far + fprintf('Octree decompression and verification succeeded!\n') ; +end diff --git a/tif_from_mj2_single_for_find_and_batch.m b/tif_from_mj2_single_for_find_and_batch.m new file mode 100644 index 0000000..7dbce61 --- /dev/null +++ b/tif_from_mj2_single_for_find_and_batch.m @@ -0,0 +1,9 @@ +function tif_from_mj2_single_for_find_and_batch(input_file_path, input_root_folder_name, output_root_folder_name) + % Decompress a single .mj2 to .tif + relative_input_file_path = relpath(input_file_path, input_root_folder_name) ; + relative_output_file_path = replace_extension(relative_input_file_path, '.tif') ; + output_file_path = fullfile(output_root_folder_name, relative_output_file_path) ; + output_folder_path = fileparts2(output_file_path) ; + ensure_folder_exists(output_folder_path) ; + tif_from_mj2_single(output_file_path, input_file_path) ; +end diff --git a/uncompress_compression_examples.m b/uncompress_compression_examples.m deleted file mode 100644 index b2e5cb7..0000000 --- a/uncompress_compression_examples.m +++ /dev/null @@ -1,5 +0,0 @@ -tif_output_folder_name = '/groups/mousebrainmicro/mousebrainmicro/data/temp/compression-examples-uncompressed' -mj2_input_folder_name = '/groups/mousebrainmicro/mousebrainmicro/data/temp/compression-examples' -do_verify = true -do_run_on_cluster = false -tif_from_mj2(tif_output_folder_name, mj2_input_folder_name, do_verify, do_run_on_cluster) diff --git a/validate_fpe.m b/validate_fpe.m deleted file mode 100644 index 48a958c..0000000 --- a/validate_fpe.m +++ /dev/null @@ -1,96 +0,0 @@ -tif_root_path = '/groups/mousebrainmicro/mousebrainmicro/data/2018-10-01' ; -mj2_root_path = '/nrs/mouselight-v/frozen/2018-10-01' ; - -tif_file_paths = find_and_list(tif_root_path, @does_match_raw_tile_tif_name); -tif_count = length(tif_file_paths) ; -shuffled_file_indices = randperm(tif_count) ; -tile_count = 50 ; -random_file_indices = shuffled_file_indices(1:tile_count) - -mj2_file_paths = cell(1,tif_count) ; -for i = 1 : tif_count , - tif_file_path = tif_file_paths{i} ; - tif_relative_path = relpath(tif_file_path, tif_root_path) ; - mj2_relative_path = replace_extension(tif_relative_path, '.mj2') ; - mj2_file_path = fullfile(mj2_root_path, mj2_relative_path) ; - mj2_file_paths{i} = mj2_file_path ; -end -%mj2_file_paths - -% load tif stacks -tic_id = tic() ; -for i = 1:tile_count , - tif_file_path = tif_file_paths{random_file_indices(i)} ; - tif_stack = read_16bit_grayscale_tif(tif_file_path) ; - if i == 1 , - tif_stacks = zeros([size(tif_stack) tile_count]) ; - end - tif_stacks(:,:,:,i) = tif_stack ; -end -toc(tic_id) ; - -% load mj2 stacks -tic_id = tic() ; -for i = 1:tile_count , - mj2_file_path = mj2_file_paths{random_file_indices(i)} ; - mj2_stack = read_16bit_grayscale_mj2(mj2_file_path) ; - if i == 1 , - mj2_stacks = zeros([size(mj2_stack) tile_count]) ; - end - mj2_stacks(:,:,:,i) = mj2_stack ; -end -toc(tic_id) ; - -tic_id = tic() ; -fpe_matrix = zeros(tile_count, tile_count) ; -for i = 1:tile_count , - tif_stack = tif_stacks(:,:,:,i) ; - for j = 1:tile_count , - mj2_stack = mj2_stacks(:,:,:,j) ; - fpe = fraction_power_explained(mj2_stack, tif_stack) ; - fpe_matrix(i,j) = fpe ; - end -end -toc(tic_id) ; -fpe_matrix - -is_on_diagonal = logical(eye(tile_count)) -is_off_diagonal = ~is_on_diagonal -fpes_on_diagonal = fpe_matrix(is_on_diagonal) -fpes_off_diagonal = fpe_matrix(is_off_diagonal) - -log_odds_fpes_on_diagonal = log(fpes_on_diagonal./(1-fpes_on_diagonal)) -log_odds_fpes_off_diagonal = log(fpes_off_diagonal./(1-fpes_off_diagonal)) - -mean_self_fpe = mean(log_odds_fpes_on_diagonal) -std_self_fpe = std(log_odds_fpes_on_diagonal) - -mean_other_fpe = mean(log_odds_fpes_off_diagonal) -std_other_fpe = std(log_odds_fpes_off_diagonal) - -self_support = min(log_odds_fpes_on_diagonal) -other_support = max(log_odds_fpes_off_diagonal) -are_separable = self_support > other_support -max_margin_threshold = (self_support + other_support)/2 - - -d_prime = (mean_self_fpe-mean_other_fpe) / sqrt( (std_self_fpe^2 + std_other_fpe^2)/2 ) - -log_odds_edges = (0:0.5:20) ; -log_odds_centers = (log_odds_edges(1:end-1)+log_odds_edges(2:end))/2 -n_self = histcounts(log_odds_fpes_on_diagonal,log_odds_edges) -n_other = histcounts(log_odds_fpes_off_diagonal,log_odds_edges) - -f = figure('color', 'w') ; -a = axes(f) -help bar -bar(log_odds_centers, n_other/sum(is_off_diagonal(:)), 'b', 'EdgeColor', 'none') -hold on -bar(log_odds_centers, n_self/sum(is_on_diagonal(:)), 'r', 'EdgeColor', 'none') -hold off -% 9 looks like a pretty good cutoff - - - - - diff --git a/validate_fve.m b/validate_fve.m deleted file mode 100644 index 6202bb2..0000000 --- a/validate_fve.m +++ /dev/null @@ -1,99 +0,0 @@ -tif_root_path = '/groups/mousebrainmicro/mousebrainmicro/data/2018-10-01' ; -mj2_root_path = '/nrs/mouselight-v/frozen/2018-10-01' ; - -tif_file_paths = find_and_list(tif_root_path, @does_match_raw_tile_tif_name); -tif_count = length(tif_file_paths) ; -shuffled_file_indices = randperm(tif_count) ; -tile_count = 3 ; -random_file_indices = shuffled_file_indices(1:tile_count) - -mj2_file_paths = cell(1,tif_count) ; -for i = 1 : tif_count , - tif_file_path = tif_file_paths{i} ; - tif_relative_path = relpath(tif_file_path, tif_root_path) ; - mj2_relative_path = replace_extension(tif_relative_path, '.mj2') ; - mj2_file_path = fullfile(mj2_root_path, mj2_relative_path) ; - mj2_file_paths{i} = mj2_file_path ; -end -%mj2_file_paths - -% load tif stacks -tic_id = tic() ; -for i = 1:tile_count , - tif_file_path = tif_file_paths{random_file_indices(i)} ; - tif_stack = read_16bit_grayscale_tif(tif_file_path) ; - if i == 1 , - tif_stacks = zeros([size(tif_stack) tile_count]) ; - end - tif_stacks(:,:,:,i) = tif_stack ; -end -toc(tic_id) ; - -% load mj2 stacks -tic_id = tic() ; -for i = 1:tile_count , - mj2_file_path = mj2_file_paths{random_file_indices(i)} ; - mj2_stack = read_16bit_grayscale_mj2(mj2_file_path) ; - if i == 1 , - mj2_stacks = zeros([size(mj2_stack) tile_count]) ; - end - mj2_stacks(:,:,:,i) = mj2_stack ; -end -toc(tic_id) ; - -tic_id = tic() ; -fpe_matrix = zeros(tile_count, tile_count) ; -for i = 1:tile_count , - tif_stack = tif_stacks(:,:,:,i) ; - for j = 1:tile_count , - mj2_stack = mj2_stacks(:,:,:,j) ; - fpe = fraction_variance_explained(mj2_stack, tif_stack) ; - fpe_matrix(i,j) = fpe ; - end -end -toc(tic_id) ; -fpe_matrix - -is_on_diagonal = logical(eye(tile_count)) -is_off_diagonal = ~is_on_diagonal -fpes_on_diagonal = fpe_matrix(is_on_diagonal) -fpes_off_diagonal = fpe_matrix(is_off_diagonal) - -neg_log_q_fpes_on_diagonal = -log(1-fpes_on_diagonal) -neg_log_q_fpes_off_diagonal = -log(1-fpes_off_diagonal) - -mean_self_fpe = mean(neg_log_q_fpes_on_diagonal) -std_self_fpe = std(neg_log_q_fpes_on_diagonal) - -mean_other_fpe = mean(neg_log_q_fpes_off_diagonal) -std_other_fpe = std(neg_log_q_fpes_off_diagonal) - -self_support = min(neg_log_q_fpes_on_diagonal) -other_support = max(neg_log_q_fpes_off_diagonal) -are_separable = self_support > other_support -max_margin_threshold = (self_support + other_support)/2 - - -d_prime = (mean_self_fpe-mean_other_fpe) / sqrt( (std_self_fpe^2 + std_other_fpe^2)/2 ) - -neg_log_q_edges = (-20:0.5:20) ; -neg_log_q_centers = (neg_log_q_edges(1:end-1)+neg_log_q_edges(2:end))/2 -n_self = histcounts(neg_log_q_fpes_on_diagonal,neg_log_q_edges) -n_other = histcounts(neg_log_q_fpes_off_diagonal,neg_log_q_edges) - -f = figure('color', 'w') ; -a = axes(f) -bar(neg_log_q_centers, n_other/sum(is_off_diagonal(:)), 'b', 'EdgeColor', 'none') -hold on -bar(neg_log_q_centers, n_self/sum(is_on_diagonal(:)), 'r', 'EdgeColor', 'none') -hold off -% anything between 0 and 1.5 looks like a pretty good cutoff -xlim([-15 +10]) ; -ylabel('Fraction of samples in class') ; -xlabel('-log(1-FVE)') ; -legend('other', 'self') - - - - - diff --git a/verify_file_sizes_after_mj2_from_tif.m b/verify_file_sizes_after_mj2_from_tif.m deleted file mode 100644 index 0a3e4ec..0000000 --- a/verify_file_sizes_after_mj2_from_tif.m +++ /dev/null @@ -1,73 +0,0 @@ -function verify_file_sizes_after_mj2_from_tif(mj2_output_folder_name, tif_input_folder_name, compression_ratio) - % Makes sure all the target files are present in the the destination, and have - % the right size, or at least a plausible size in the case of .mj2's. - is_all_well = true ; - is_all_well = verify_file_sizes_after_mj2_from_tif_helper(mj2_output_folder_name, tif_input_folder_name, compression_ratio, is_all_well) ; - if is_all_well , - fprintf('Target folder %s matches source folder %s\n', mj2_output_folder_name, tif_input_folder_name) ; - else - error('Target folder %s had discrepancies', mj2_output_folder_name) ; - end -end - - - -function is_all_well = verify_file_sizes_after_mj2_from_tif_helper(mj2_output_folder_name, tif_input_folder_name, compression_ratio, is_all_well) - tif_input_entities = dir_without_dot_and_dot_dot(tif_input_folder_name) ; - tif_input_entity_count = length(tif_input_entities) ; - for i = 1 : tif_input_entity_count , - tif_input_entity = tif_input_entities(i) ; - tif_input_entity_name = tif_input_entity.name ; - tif_input_entity_path = fullfile(tif_input_folder_name, tif_input_entity_name) ; - if tif_input_entity.isdir , - % if a folder, recurse - mj2_output_entity_path = fullfile(mj2_output_folder_name, tif_input_entity_name) ; - is_all_well = verify_file_sizes_after_mj2_from_tif_helper(mj2_output_entity_path, tif_input_entity_path, compression_ratio, is_all_well) ; - else - % If a normal file, check the file size - [~,~,ext] = fileparts(tif_input_entity_path) ; - if isequal(ext, '.tif') , - mj2_output_entity_path = fullfile(mj2_output_folder_name, replace_extension(tif_input_entity_name, '.mj2')) ; - if exist(mj2_output_entity_path, 'file') , - tif_file_size = tif_input_entity.bytes ; - mj2_file_size = get_file_size(mj2_output_entity_path) ; - desired_mj2_file_size = round(tif_file_size/compression_ratio) ; - if mj2_file_size == desired_mj2_file_size , - % do nothing, this is fine and good, and should handle zero-length files - elseif round(0.8*desired_mj2_file_size)<=mj2_file_size && mj2_file_size<=round(1.2*desired_mj2_file_size) - % do nothing this is also fine - else - is_all_well = false ; - fprintf('Target file %s is %d bytes, which seems suspect given the desired size was %d bytes\n', ... - mj2_output_entity_path, ... - mj2_file_size, ... - desired_mj2_file_size) ; - end - else - is_all_well = false ; - fprintf('Target file %s is missing\n', mj2_output_entity_path) ; - end - else - % Source file is not a tif, so destination file should be exactly the same size - % as source. - mj2_output_entity_path = fullfile(mj2_output_folder_name, tif_input_entity_name) ; % input not really a tif, output not really a .mj2 - if exist(mj2_output_entity_path, 'file') , - tif_file_size = tif_input_entity.bytes ; - mj2_file_size = get_file_size(mj2_output_entity_path) ; - if mj2_file_size == tif_file_size , - % do nothing, this is fine and good - else - is_all_well = false ; - fprintf('Target file %s is %d bytes, but it should be %d bytes\n', ... - mj2_output_entity_path, ... - mj2_file_size, ... - tif_file_size) ; - end - else - is_all_well = false ; - fprintf('Target file %s is missing\n', mj2_output_entity_path) ; - end - end - end - end -end diff --git a/verify_file_sizes_for_2019_03_26_frozen_raw_tiles.m b/verify_file_sizes_for_2019_03_26_frozen_raw_tiles.m deleted file mode 100644 index d9f206a..0000000 --- a/verify_file_sizes_for_2019_03_26_frozen_raw_tiles.m +++ /dev/null @@ -1,5 +0,0 @@ -sample_date = '2019-03-26' ; -mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -compression_ratio = 10 ; -verify_file_sizes_after_mj2_from_tif(mj2_output_folder_path, tif_input_folder_path, compression_ratio) diff --git a/verify_files_after_mj2_from_tif.m b/verify_files_after_mj2_from_tif.m deleted file mode 100644 index 7dbf6b0..0000000 --- a/verify_files_after_mj2_from_tif.m +++ /dev/null @@ -1,74 +0,0 @@ -function verify_files_after_mj2_from_tif(mj2_output_folder_name, tif_input_folder_name) - % Makes sure all the target files are present in the the destination, and have - % the right size, or at least a plausible size in the case of .mj2's. - is_all_well = true ; - is_all_well = verify_files_after_mj2_from_tif_helper(mj2_output_folder_name, tif_input_folder_name, is_all_well) ; - if is_all_well , - fprintf('Target folder %s matches source folder %s\n', mj2_output_folder_name, tif_input_folder_name) ; - else - error('Target folder %s had discrepancies', mj2_output_folder_name) ; - end -end - - - -function is_all_well = verify_files_after_mj2_from_tif_helper(mj2_output_folder_name, tif_input_folder_name, is_all_well) - tif_input_entities = dir_without_dot_and_dot_dot(tif_input_folder_name) ; - tif_input_entity_count = length(tif_input_entities) ; - for i = 1 : tif_input_entity_count , - tif_input_entity = tif_input_entities(i) ; - tif_input_entity_name = tif_input_entity.name ; - tif_input_entity_path = fullfile(tif_input_folder_name, tif_input_entity_name) ; - if tif_input_entity.isdir , - % if a folder, recurse - mj2_output_entity_path = fullfile(mj2_output_folder_name, tif_input_entity_name) ; - is_all_well = verify_files_after_mj2_from_tif_helper(mj2_output_entity_path, tif_input_entity_path, is_all_well) ; - else - [~,~,ext] = fileparts(tif_input_entity_path) ; - if isequal(ext, '.tif') , - % If source is a tif, read in both files, and make sure they are close to one - % another. - mj2_output_entity_path = fullfile(mj2_output_folder_name, replace_extension(tif_input_entity_name, '.mj2')) ; - if exist(mj2_output_entity_path, 'file') , - try - mj2_stack = read_16bit_grayscale_mj2(mj2_output_entity_path) ; - catch err - is_all_well = false ; - fprintf('Unable to read target file %s: %s\n', mj2_output_entity_path, err.message) ; - end - tif_stack = read_16bit_grayscale_tif(tif_input_entity_path) ; - if is_mj2_similar_to_tif(mj2_stack, tif_stack) , - % do nothing, all is well - else - is_all_well = false ; - fprintf('Target file %s is considerably different from source file%s\n', ... - mj2_output_entity_path, ... - tif_input_entity_path) ; - end - else - is_all_well = false ; - fprintf('Target file %s is missing\n', mj2_output_entity_path) ; - end - else - % Source file is not a tif, so destination file should be exactly the same - % as source. - mj2_output_entity_path = fullfile(mj2_output_folder_name, tif_input_entity_name) ; % input not really a tif, output not really a .mj2 - if exist(mj2_output_entity_path, 'file') , - tif_file_contents = read_file_into_uint8_array(tif_input_entity_path) ; - mj2_file_contents = read_file_into_uint8_array(mj2_output_entity_path) ; - if isequal(tif_file_contents, mj2_file_contents) , - % do nothing, this is fine and good - else - is_all_well = false ; - fprintf('Target file %s differs from source file %s\n', ... - mj2_output_entity_path, ... - tif_input_entity_path) ; - end - else - is_all_well = false ; - fprintf('Target file %s is missing\n', mj2_output_entity_path) ; - end - end - end - end -end diff --git a/verify_frozen_2018_07_02.m b/verify_frozen_2018_07_02.m deleted file mode 100644 index 41e0249..0000000 --- a/verify_frozen_2018_07_02.m +++ /dev/null @@ -1,5 +0,0 @@ -sample_date = '2018-07-02' ; -mj2_root_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -tif_root_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; - -is_all_well = verify_frozen_files(mj2_root_folder_path, tif_root_folder_path) diff --git a/verify_frozen_2019_04_17.m b/verify_frozen_2019_04_17.m deleted file mode 100644 index 48058e3..0000000 --- a/verify_frozen_2019_04_17.m +++ /dev/null @@ -1,5 +0,0 @@ -sample_date = '2019-04-17' ; -mj2_root_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -tif_root_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; - -is_all_well = verify_frozen_files(mj2_root_folder_path, tif_root_folder_path) diff --git a/verify_frozen_files.m b/verify_frozen_files.m deleted file mode 100644 index ad41d3c..0000000 --- a/verify_frozen_files.m +++ /dev/null @@ -1,17 +0,0 @@ -function result = verify_frozen_files(mj2_root_folder_path, tif_root_folder_path, do_submit) - fprintf('Doing batch preverification of large files...\n') ; - batch_preverify_tif_files_after_mj2_from_tif(mj2_root_folder_path, tif_root_folder_path, do_submit) - fprintf('Done with batch preverification of large files.\n') ; - - fprintf('Doing verification of large files...\n') ; - is_all_well_with_big_files = verify_tif_files_after_mj2_from_tif(tif_root_folder_path, mj2_root_folder_path) ; - fprintf('Done with verification of large files.\n') ; - - if is_all_well_with_big_files, - fprintf('Doing verification of small files...\n') ; - result = verify_non_tif_files_after_mj2_from_tif(mj2_root_folder_path, tif_root_folder_path) ; - fprintf('Done with verification of small files.\n') ; - else - result = false ; - end -end diff --git a/verify_non_tif_files_after_thawing.m b/verify_non_tif_files_after_thawing.m new file mode 100644 index 0000000..87621b8 --- /dev/null +++ b/verify_non_tif_files_after_thawing.m @@ -0,0 +1,10 @@ +function verify_non_tif_files_after_thawing(output_root_folder_path, input_root_folder_path) + % Makes sure all the output files are present in the the destination, and have + % the same size and md5sum. + + find_and_verify(input_root_folder_path, ... + @does_file_name_not_end_in_dot_mj2, ... + @verify_single_non_stack_file, ... + input_root_folder_path, ... + output_root_folder_path) ; +end diff --git a/verify_non_tif_files_for_2019_03_26_frozen_raw_tiles.m b/verify_non_tif_files_for_2019_03_26_frozen_raw_tiles.m deleted file mode 100644 index 13d98c6..0000000 --- a/verify_non_tif_files_for_2019_03_26_frozen_raw_tiles.m +++ /dev/null @@ -1,4 +0,0 @@ -sample_date = '2019-03-26' ; -mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -verify_non_tif_files_after_mj2_from_tif(mj2_output_folder_path, tif_input_folder_path) diff --git a/verify_single_non_stack_file.m b/verify_single_non_stack_file.m new file mode 100644 index 0000000..ac9c38b --- /dev/null +++ b/verify_single_non_stack_file.m @@ -0,0 +1,10 @@ +function result = verify_single_non_stack_file(input_file_path, input_root_folder_path, output_root_folder_path) + % Compare the md5sum of + relative_file_path = relpath(input_file_path, input_root_folder_path) ; + output_file_path = fullfile(output_root_folder_path, relative_file_path) ; + if exist(output_file_path, 'file') , + result = are_files_same_size_and_md5sum(input_file_path, output_file_path) ; + else + result = false ; + end +end diff --git a/verify_single_non_tif_file_after_mj2_from_tif.m b/verify_single_non_tif_file_after_mj2_from_tif.m index 89f470d..ca7946a 100644 --- a/verify_single_non_tif_file_after_mj2_from_tif.m +++ b/verify_single_non_tif_file_after_mj2_from_tif.m @@ -1,11 +1,9 @@ -function result = verify_single_non_tif_file_after_mj2_from_tif(tif_file_path, tif_root_folder_path, mj2_root_folder_path) - % The two files must be identical - relative_file_path = relpath(tif_file_path, tif_root_folder_path) ; - mj2_file_path = fullfile(mj2_root_folder_path, relative_file_path) ; - if exist(mj2_file_path, 'file') , - tif_file_contents = read_file_into_uint8_array(tif_file_path) ; - mj2_file_contents = read_file_into_uint8_array(mj2_file_path) ; - result = isequal(tif_file_contents, mj2_file_contents) ; +function result = verify_single_non_tif_file_after_mj2_from_tif(tif_side_file_path, tif_root_folder_path, mj2_root_folder_path) + % Compare the md5sum of + relative_file_path = relpath(tif_side_file_path, tif_root_folder_path) ; + mj2_side_file_path = fullfile(mj2_root_folder_path, relative_file_path) ; + if exist(mj2_side_file_path, 'file') , + result = are_files_same_size_and_md5sum(tif_side_file_path, mj2_side_file_path) ; else result = false ; end diff --git a/verify_single_small_file_after_mj2_from_tif.m b/verify_single_small_file_after_mj2_from_tif.m deleted file mode 100644 index bae13ec..0000000 --- a/verify_single_small_file_after_mj2_from_tif.m +++ /dev/null @@ -1,12 +0,0 @@ -function result = verify_single_small_file_after_mj2_from_tif(tif_file_path, tif_root_folder_path, mj2_root_folder_path) - % The two files must be identical - relative_file_path = relpath(tif_file_path, tif_root_folder_path) ; - mj2_file_path = fullfile(mj2_root_folder_path, relative_file_path) ; - if exist(mj2_file_path, 'file') , - tif_file_contents = read_file_into_uint8_array(tif_file_path) ; - mj2_file_contents = read_file_into_uint8_array(mj2_file_path) ; - result = isequal(tif_file_contents, mj2_file_contents) ; - else - result = false ; - end -end diff --git a/verify_single_tif_file_after_thawing.m b/verify_single_tif_file_after_thawing.m new file mode 100644 index 0000000..912dadd --- /dev/null +++ b/verify_single_tif_file_after_thawing.m @@ -0,0 +1,29 @@ +function verify_single_tif_file_after_thawing(input_file_path, input_root_folder_name, output_root_folder_name) + % Checks that the output .tif is similar to the .mj2, and outputs a + % .is-similar-to-tif file in the output folder if they are similar. If able + % to run verification, but similarity is below threshold, writes a + % .is-not-similar-to-tif file. If output file is missing or unreadable, writes + % nothing. + input_relative_file_path = relpath(input_file_path, input_root_folder_name) ; + output_relative_file_path = replace_extension(input_relative_file_path, '.tif') ; + output_file_path = fullfile(output_root_folder_name, output_relative_file_path) ; + cert_file_path = horzcat(output_file_path, '.is-similar-to-mj2') ; + not_check_file_path = horzcat(output_file_path, '.is-not-similar-to-mj2') ; + if exist(cert_file_path, 'file') , + return + end + if exist(output_file_path, 'file') , + try + output_stack = read_16bit_grayscale_tif(output_file_path) ; + catch err %#ok + return + end + input_stack = read_16bit_grayscale_mj2(input_file_path) ; + fpe_value = fraction_variance_explained(output_stack, input_stack) ; + if fpe_value > 0.75 , + write_string_to_file(cert_file_path, sprintf('%.10f\n', fpe_value)) ; + else + write_string_to_file(not_check_file_path, sprintf('%.10f\n', fpe_value)) ; + end + end +end diff --git a/verify_tif_files_after_mj2_from_tif.m b/verify_tif_files_after_mj2_from_tif.m index cb34b96..8462979 100644 --- a/verify_tif_files_after_mj2_from_tif.m +++ b/verify_tif_files_after_mj2_from_tif.m @@ -1,10 +1,10 @@ function verify_tif_files_after_mj2_from_tif(tif_root_folder_path, mj2_root_folder_path) % Makes sure there's a .similar-mj2-exists for each .tif file in - % tif_root_folder_path. - find_and_verify(tif_root_folder_path, ... - @does_file_name_end_in_dot_tif, ... - @verify_single_tif_file_after_mj2_from_tif, ... - tif_root_folder_path, ... - mj2_root_folder_path) ; + % tif_root_folder_path. + find_and_verify(... + tif_root_folder_path, ... + @does_file_name_end_in_dot_tif, ... + @verify_single_tif_file_after_mj2_from_tif, ... + tif_root_folder_path, ... + mj2_root_folder_path) ; end - \ No newline at end of file diff --git a/verify_tif_files_for_2019_03_26_frozen_raw_tiles.m b/verify_tif_files_for_2019_03_26_frozen_raw_tiles.m deleted file mode 100644 index 36dcaa2..0000000 --- a/verify_tif_files_for_2019_03_26_frozen_raw_tiles.m +++ /dev/null @@ -1,4 +0,0 @@ -sample_date = '2019-03-26' ; -mj2_output_folder_path = sprintf('/nrs/mouselight-v/frozen/%s', sample_date) ; -tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ; -post_batch_verify_tif_files_after_mj2_from_tif(mj2_output_folder_path, tif_input_folder_path) diff --git a/voxelwise_mean_relative_error_with_guard.m b/voxelwise_mean_relative_error_with_guard.m deleted file mode 100644 index bb0571e..0000000 --- a/voxelwise_mean_relative_error_with_guard.m +++ /dev/null @@ -1,16 +0,0 @@ -function result = voxelwise_mean_relative_error_with_guard(mj2_stack, tif_stack, voxel_scale) - if isequal(class(mj2_stack), class(tif_stack)) && isequal(size(mj2_stack), size(tif_stack)) , - if all(all(all(mj2_stack==tif_stack))) , - result = 0 ; - else - mj2_stack_serial = double(mj2_stack(:))/(2^16-1) ; - tif_stack_serial = double(tif_stack(:))/(2^16-1) ; - absolute_error = abs(mj2_stack_serial-tif_stack_serial) ; - error_scale = max(voxel_scale, tif_stack_serial) ; % voxel scale of 1/256 is what I've used for uint16 data... - summary_error = absolute_error ./ error_scale ; % relative error, but with a guard against very small target values - result = mean(summary_error) ; - end - else - result = +inf ; - end -end diff --git a/why_are_fve_values_so_low.m b/why_are_fve_values_so_low.m deleted file mode 100644 index 078f79c..0000000 --- a/why_are_fve_values_so_low.m +++ /dev/null @@ -1,53 +0,0 @@ -tif_root_path = '/groups/mousebrainmicro/mousebrainmicro/data/2018-10-01' ; -mj2_root_path = '/nrs/mouselight-v/frozen/2018-10-01' ; - -tif_file_paths = find_and_list(tif_root_path, @does_match_raw_tile_tif_name); -tif_count = length(tif_file_paths) ; -shuffled_file_indices = randperm(tif_count) ; -tile_count = 20 ; -random_file_indices = shuffled_file_indices(1:tile_count) -%random_file_indices = [ 16503 35805 9320] ; - -mj2_file_paths = cell(1,tif_count) ; -for i = 1 : tif_count , - tif_file_path = tif_file_paths{i} ; - tif_relative_path = relpath(tif_file_path, tif_root_path) ; - mj2_relative_path = replace_extension(tif_relative_path, '.mj2') ; - mj2_file_path = fullfile(mj2_root_path, mj2_relative_path) ; - mj2_file_paths{i} = mj2_file_path ; -end -%mj2_file_paths - -% compute fve for each -tic_id = tic() ; -fves_on_diagonal = zeros(tile_count, 1) ; -for i = 1:tile_count , - tif_file_path = tif_file_paths{random_file_indices(i)} ; - tif_stack = read_16bit_grayscale_tif(tif_file_path) ; - mj2_file_path = mj2_file_paths{random_file_indices(i)} ; - mj2_stack = read_16bit_grayscale_mj2(mj2_file_path) ; - fves_on_diagonal(i) = fraction_variance_explained(mj2_stack, tif_stack) ; -end -toc(tic_id) ; -fves_on_diagonal - - -% % 2 has the lowest fve -% tif_file_path = tif_file_paths{random_file_indices(2)} -% mj2_file_path = mj2_file_paths{random_file_indices(2)} -% mj2_stack = mj2_stacks(:,:,:,2) ; -% tif_stack = tif_stacks(:,:,:,2) ; -% self_difference_stack = mj2_stack - tif_stack ; -% abs_diff_stack = abs(self_difference_stack) ; -% [max_abs_diff, i_max_abs_diff] = max(abs_diff_stack(:)) -% [i_mad,j_mad,k_mad] = ind2sub(size(abs_diff_stack), i_max_abs_diff) -% -% mj2_slice = mj2_stack(:,:,k_mad) ; -% tif_slice = tif_stack(:,:,k_mad) ; -% - -% has the highest fve -[max_fve, i_max_fve] = max(fves_on_diagonal) -file_index = random_file_indices(i_max_fve) -tif_file_path = tif_file_paths{file_index} -mj2_file_path = mj2_file_paths{file_index}