Added 'recent' top-level scripts

MouseLightProject · Mar 31, 2023 · bba2569 · bba2569
1 parent 6259a35
commit bba2569
Show file tree

Hide file tree

Showing 28 changed files with 168 additions and 7 deletions.
diff --git a/.gitignore b/.gitignore
@@ -12,4 +12,5 @@ freezing-test*
 thawing-test*
 *.mat
 *.out.txt
+*.out2.txt
 nohup.out
diff --git a/_template_driver_script.sh b/_template_driver_script.sh
@@ -1,3 +1,3 @@
 cd /groups/mousebrainmicro/mousebrainmicro/scripts/freezing
-out=freeze-2021-04-10.out.txt
-bsub -P mouselight -n 8 -oo $out -eo $out /misc/local/matlab-2019a/bin/matlab -nodisplay -batch "modpath; freeze_2021_04_10"
+out=freeze-2020-12-31.out.txt
+bsub -P mouselight -n 8 -oo $out -eo $out /misc/local/matlab-2019a/bin/matlab -nodisplay -batch "modpath; freeze_2020_12_31"
diff --git a/freeze_2019_08_22.m b/freeze_2019_08_22.m
@@ -0,0 +1,8 @@
+sample_date = '2019-08-22' ;
+mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/RAW_archive/%s', sample_date) ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_2020_10_16.m b/freeze_2020_10_16.m
@@ -0,0 +1,8 @@
+sample_date = '2020-10-16' ;
+mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/RAW_archive/%s', sample_date) ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_2020_10_16_driver_script.sh b/freeze_2020_10_16_driver_script.sh
@@ -0,0 +1,3 @@
+cd /groups/mousebrainmicro/mousebrainmicro/scripts/freezing/
+out=freeze-2022-10-16.out.txt
+bsub -P mouselight -n 8 -oo $out -eo $out /misc/local/matlab-2019a/bin/matlab -nodisplay -batch "modpath; freeze_2020_10_16"
diff --git a/freeze_2020_11_26_compression_test.m b/freeze_2020_11_26_compression_test.m
@@ -0,0 +1,8 @@
+sample_date = '2020-11-26' ;
+mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/test-data/test-unscaled/compression-tests/%s', sample_date) ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/test-data/test-unscaled/2020-11-26/') ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_2020_12_31.m b/freeze_2020_12_31.m
@@ -1,9 +1,8 @@
-sample_date = '2020-09-15' ;
-mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/%s', sample_date) ;
-tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/%s', sample_date) ;
+sample_date = '2020-12-31' ;
+mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/RAW_archive/%s', sample_date) ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ;
 compression_ratio = 10 ;
 do_run_on_cluster = true ;
 maximum_running_slot_count = 500 ;
 
 freeze_sample
-
diff --git a/freeze_2020_12_31_driver_script.sh b/freeze_2020_12_31_driver_script.sh
@@ -0,0 +1,3 @@
+cd /groups/mousebrainmicro/mousebrainmicro/scripts/freezing/
+out=freeze-2022-12-31.out.txt
+bsub -P mouselight -n 8 -oo $out -eo $out /misc/local/matlab-2019a/bin/matlab -nodisplay -batch "modpath; freeze_2020_12_31"
diff --git a/freeze_2021_02_01.m b/freeze_2021_02_01.m
@@ -0,0 +1,8 @@
+sample_date = '2022-05-10' ;
+mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_2021_07_20.m b/freeze_2021_07_20.m
@@ -0,0 +1,7 @@
+mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/2021-07-20/') ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_render/2021-07-20/Tiling/') ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_2021_07_20_alt.m b/freeze_2021_07_20_alt.m
@@ -0,0 +1,7 @@
+mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/2021-07-20') ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_render/2021-07-20/Tiling') ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_2021_07_20_debug.m b/freeze_2021_07_20_debug.m
@@ -0,0 +1,7 @@
+mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/2021-07-20/') ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_render/2021-07-20/Tiling/') ;
+compression_ratio = 10 ;
+do_run_on_cluster = false ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_2021_07_20b.m b/freeze_2021_07_20b.m
@@ -0,0 +1,8 @@
+sample_date = '2021-07-20' ;
+mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/RAW_archive/%s', sample_date) ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_2021_07_20b_driver_script.sh b/freeze_2021_07_20b_driver_script.sh
@@ -0,0 +1,3 @@
+cd /groups/mousebrainmicro/mousebrainmicro/scripts/freezing/
+out=freeze-2022-07-20b.out.txt
+bsub -P mouselight -n 8 -oo $out -eo $out /misc/local/matlab-2019a/bin/matlab -nodisplay -batch "modpath; freeze_2021_07_20b"
diff --git a/freeze_2021_09_16.m b/freeze_2021_09_16.m
@@ -0,0 +1,8 @@
+sample_date = '2021-09-16' ;
+mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_2021_10_07.m b/freeze_2021_10_07.m
@@ -0,0 +1,8 @@
+sample_date = '2021-10-07' ;
+mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_2021_10_19.m b/freeze_2021_10_19.m
@@ -0,0 +1,8 @@
+sample_date = '2021-10-19' ;
+mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/RAW_archive/%s', sample_date) ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_2021_12_17.m b/freeze_2021_12_17.m
@@ -0,0 +1,8 @@
+sample_date = '2021-12-17' ;
+mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_2022_02_01.m b/freeze_2022_02_01.m
@@ -0,0 +1,8 @@
+sample_date = '2022-02-01' ;
+mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_2022_03_23.m b/freeze_2022_03_23.m
@@ -0,0 +1,8 @@
+sample_date = '2022-03-23' ;
+mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_2022_04_25.m b/freeze_2022_04_25.m
@@ -0,0 +1,8 @@
+sample_date = '2022-04-25' ;
+mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_2022_05_10.m b/freeze_2022_05_10.m
@@ -0,0 +1,8 @@
+sample_date = '2022-05-10' ;
+mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_2022_05_30.m b/freeze_2022_05_30.m
@@ -0,0 +1,8 @@
+sample_date = '2022-05-30' ;
+mj2_output_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_nearline_raw/%s', sample_date) ;
+tif_input_folder_path = sprintf('/groups/mousebrainmicro/mousebrainmicro/data/to_compress/%s', sample_date) ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/freeze_temp.m b/freeze_temp.m
@@ -0,0 +1,8 @@
+sample_date = '2020-04-15' ;
+mj2_output_folder_path = sprintf('/nrs/mouselight/SAMPLES/TO_NEARLINE/RAW_archive/%s', sample_date) ;
+tif_input_folder_path = sprintf('/nrs/mouselight/pipeline_output/%s/stage_1_line_fix_output/', sample_date) ;
+compression_ratio = 10 ;
+do_run_on_cluster = true ;
+maximum_running_slot_count = 500 ;
+
+freeze_sample
diff --git a/mj2_from_tif.m b/mj2_from_tif.m
@@ -71,6 +71,7 @@ function mj2_from_tif_helper_bang(bqueue, bsub_options, slots_per_job, mj2_folde
 %                                       mj2_file_path, ...
 %                                       tif_file_path, ...
 %                                       compression_ratio) ;
+                    fprintf('Enqueuing job for %s\n', tif_file_path) ;
                     bqueue.enqueue(slots_per_job, ...
                                    [], ...
                                    bsub_options, ...

diff --git a/quickhydrate_2018_03_09.m b/quickhydrate_2018_03_09.m
@@ -0,0 +1,5 @@
+do_verify = true ;
+do_on_cluster = false ;
+addpath('/groups/mousebrainmicro/mousebrainmicro/scripts/freezing/')  
+modpath ;
+tif_from_mj2('/groups/scicompsoft/home/schauderd/mj2','/groups/scicompsoft/home/schauderd/mj2', do_verify, do_on_cluster) ;
diff --git a/tmt b/tmt
diff --git a/uncompress_compression_examples.m b/uncompress_compression_examples.m
@@ -0,0 +1,5 @@
+tif_output_folder_name = '/groups/mousebrainmicro/mousebrainmicro/data/temp/compression-examples-uncompressed'
+mj2_input_folder_name = '/groups/mousebrainmicro/mousebrainmicro/data/temp/compression-examples' 
+do_verify = true 
+do_run_on_cluster = false
+tif_from_mj2(tif_output_folder_name, mj2_input_folder_name, do_verify, do_run_on_cluster)
+4 −0		basics/add_ones_column.m
+5 −0		basics/add_ones_row.m
+26 −0		basics/colormaps/bwr.m
+4 −4		basics/colormaps/oppo.m
+26 −0		basics/colormaps/rwb.m
+6 −6		basics/filesystemy/dirwalk.m
+6 −0		basics/filesystemy/ensure_file_does_not_exist.m
+20 −0		basics/filesystemy/ensure_folder_exists.m
+6 −0		basics/filesystemy/escape_string_for_bash.m
+31 −0		basics/filesystemy/file_object.m
+4 −0		basics/filesystemy/fileparts2.m
+3 −1		basics/filesystemy/read_file_into_cellstring.m
+16 −0		basics/filesystemy/read_file_into_cellstring_keeping_newlines.m
+9 −0		basics/filesystemy/read_file_into_char_array.m
+8 −0		basics/filesystemy/read_file_into_uint8_array.m
+7 −0		basics/filesystemy/touch.m
+13 −0		basics/get_short_host_name.m
+77 −0		basics/set_parpool_job_storage_location.m
+18 −0		basics/str2logical.m
+28 −0		basics/system_from_list_with_error_handling.m
+0 −0		fif.m
+8 −1		fuster/bqueue_type.m
+20 −5		fuster/bsub.m
+2 −1		fuster/get_bjobs_lines.m
+10 −7		fuster/get_bsub_job_status.m
+0 −49		fuster/get_bsub_job_status_new.m
+6 −3		fuster/get_single_bsub_job_status.m
+2 −1		fuster/test_bqueue.m
+11 −0		mouselighty/affine_from_cpg_targets.m
+13 −0		mouselighty/are_paired_descriptors_close_enough.m
+60 −0		mouselighty/barycentric_interpolant.m
+14 −6		mouselighty/build_raw_tile_index.m
+13 −0		mouselighty/channel_index0_as_string_from_tile_file_name.m
+1 −1		mouselighty/chunk_ijk0_from_bits.m
+2 −3		mouselighty/chunk_ijk1s_from_octree_paths.m
+7 −2		mouselighty/collect_z_face_matches.m
+41 −0		mouselighty/compute_affine_landmark_match_distance.m
+74 −0		mouselighty/compute_barycentric_landmark_match_distance.m
+71 −0		mouselighty/compute_barycentric_landmark_match_error.m
+72 −0		mouselighty/find_shift.m
+96 −0		mouselighty/fix_line_shift.m
+5 −2		mouselighty/imagery_file_relative_path_from_relative_path.m
+8 −0		mouselighty/landmark_file_name_from_tile_relative_path.m
+93 −0		mouselighty/read_sample_metadata_file.m
+1 −1		mouselighty/read_xlineshift_file.m
+6 −2		mouselighty/simple_dir.m
+13 −0		mouselighty/tile_relative_path_from_tile_file_path.m
+99 −0		mouselighty/writeYML.m
+8 −0		mouselighty/write_xlineshift_file.m