WIP: changing script 🐛

neuroinformatics-unit · Dec 4, 2024 · 76ff8b7 · 76ff8b7
1 parent 0e3821b
commit 76ff8b7
Show file tree

Hide file tree

Showing 4 changed files with 69 additions and 73 deletions.
diff --git a/_datasets.csv b/_datasets.csv
@@ -0,0 +1,21 @@
+index,read_dataset_path,write_dataset_path
+0,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230804_CAA_1119917,/ceph/margrie/laura/cimaut/derivatives/sub-0_230804_CAA_1119917
+1,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230818_CAA_1120210,/ceph/margrie/laura/cimaut/derivatives/sub-1_230818_CAA_1120210
+2,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230803_CAA_1119915,/ceph/margrie/laura/cimaut/derivatives/sub-2_230803_CAA_1119915
+3,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230801_CAA_1120181,/ceph/margrie/laura/cimaut/derivatives/sub-3_230801_CAA_1120181
+4,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230802_CAA_1120182,/ceph/margrie/laura/cimaut/derivatives/sub-4_230802_CAA_1120182
+5,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230803_CAA_1120181,/ceph/margrie/laura/cimaut/derivatives/sub-5_230803_CAA_1120181
+6,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230822_CAA_1120509,/ceph/margrie/laura/cimaut/derivatives/sub-6_230822_CAA_1120509
+7,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230823_CAA_1120181,/ceph/margrie/laura/cimaut/derivatives/sub-7_230823_CAA_1120181
+8,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230824_CAA_1119915,/ceph/margrie/laura/cimaut/derivatives/sub-8_230824_CAA_1119915
+9,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230825_CAA_1120182,/ceph/margrie/laura/cimaut/derivatives/sub-9_230825_CAA_1120182
+10,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230905_CAA_1119917,/ceph/margrie/laura/cimaut/derivatives/sub-10_230905_CAA_1119917
+11,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230907_CAA_1120210,/ceph/margrie/laura/cimaut/derivatives/sub-11_230907_CAA_1120210
+12,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230907_CAA_1120509,/ceph/margrie/laura/cimaut/derivatives/sub-12_230907_CAA_1120509
+13,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230912_CAA_1119915,/ceph/margrie/laura/cimaut/derivatives/sub-13_230912_CAA_1119915
+14,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230912_CAA_1120051,/ceph/margrie/laura/cimaut/derivatives/sub-14_230912_CAA_1120051
+15,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230913_CAA_1120182,/ceph/margrie/laura/cimaut/derivatives/sub-15_230913_CAA_1120182
+16,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230913_CAA_1120395,/ceph/margrie/laura/cimaut/derivatives/sub-16_230913_CAA_1120395
+17,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230914_CAA_1120181,/ceph/margrie/laura/cimaut/derivatives/sub-17_230914_CAA_1120181
+18,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230914_CAA_1120210,/ceph/margrie/laura/cimaut/derivatives/sub-18_230914_CAA_1120210
+19,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230915_CAA_1120509,/ceph/margrie/laura/cimaut/derivatives/sub-19_230915_CAA_1120509
diff --git a/calcium_imaging_automation/core/rules/preprocess.py b/calcium_imaging_automation/core/rules/preprocess.py
@@ -10,8 +10,6 @@
     write_dataset_path = Path(snakemake.input[1])
     output = snakemake.output[0]
 
-    # os.system("module load miniconda")
-    # os.system("source activate /nfs/nhome/live/lporta/.conda/envs/cimat")
     output_path_dataset = write_dataset_path / "ses-0/funcimg/"
 
     data = derotate(read_dataset_path, output_path_dataset)

diff --git a/calcium_imaging_automation/core/rules/setup.py b/calcium_imaging_automation/core/rules/setup.py
@@ -6,19 +6,27 @@
 
 from calcium_imaging_automation.core.reader import ReadAquiredData
 from calcium_imaging_automation.core.writer import DatashuttleWrapper
+from snakemake.script import snakemake
 
 
-def setup(raw_data_path, folder_read_pattern, file_read_pattern, output_path):
+try:
+    read_dataset_path = Path(snakemake.input[0])
+    write_dataset_path = Path(snakemake.input[1])
+    folder_read_pattern = snakemake.params.folder_read_pattern
+    file_read_pattern = snakemake.params.file_read_pattern
+
+    output = snakemake.output[0]
+
     try:
         shutil.rmtree("/ceph/margrie/laura/cimaut/derivatives/")
         shutil.rmtree("/ceph/margrie/laura/cimaut/submitit/")
     except FileNotFoundError:
         print("No derivatives folder found")
 
-    print(f"Reading data from {raw_data_path}")
+    print(f"Reading data from {read_dataset_path}")
 
     reader = ReadAquiredData(
-        raw_data_path,
+        read_dataset_path,
         folder_read_pattern,
         file_read_pattern,
     )
@@ -27,7 +35,7 @@ def setup(raw_data_path, folder_read_pattern, file_read_pattern, output_path):
     number_of_tiffs = reader.max_session_number(filetype="tif")
     print(f"Max of tiffs found: {number_of_tiffs}")
 
-    writer = DatashuttleWrapper(output_path)
+    writer = DatashuttleWrapper(write_dataset_path)
     writer.create_folders(reader.dataset_names, session_number=number_of_tiffs)
     print("Folders created")
 
@@ -40,44 +48,9 @@ def setup(raw_data_path, folder_read_pattern, file_read_pattern, output_path):
             ],
         }
     )
-    datasets.to_csv("datasets.csv", index=True, index_label="index")
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description="Example usage of the pipeline manager."
-    )
-
-    parser.add_argument(
-        "raw_data_path", type=Path, help="Path to the raw data."
-    )
-    parser.add_argument(
-        "output_path", type=Path, help="Path to the output data."
-    )
-    parser.add_argument(
-        "--folder_read_pattern",
-        type=str,
-        help="Glob pattern for reading folder.",
-        default="*",
-    )
-    parser.add_argument(
-        "--file_read_pattern",
-        type=str,
-        help="List of glob patterns for reading files.",
-        action="append",
-    )
-
-    args = parser.parse_args()
-
-    try:
-        setup(
-            args.raw_data_path,
-            args.folder_read_pattern,
-            args.file_read_pattern,
-            args.output_path,
-        )
-
-        print("Success")
-    except Exception as e:
-        print(f"Error: {e.args}")
-        print(e.with_traceback(e.__traceback__))
+    datasets.to_csv(output, index=True, index_label="index")
+
+except Exception as e:
+    print(e.args)
+    with open(output, "w") as f:
+        f.write(str(e.args))
diff --git a/workflow/Snakefile b/workflow/Snakefile
@@ -1,31 +1,35 @@
 rule setup:
     input:
-        datasets_path="/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/",
-        writing_path="/ceph/margrie/laura/cimaut/",
-    output: "setup_output.txt"
-    shell: "python calcium_imaging_automation/core/rules/setup.py {input.datasets_path} {input.writing_path} --folder_read_pattern '2*' --file_read_pattern 'rotation_00001.tif' --file_read_pattern '*.bin' > {output}"
-
-import pandas as pd
+        "/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/",
+        "/ceph/margrie/laura/cimaut/",
+    params:
+        folder_read_pattern="2*",
+        file_read_pattern=["rotation_00001.tif", "*.bin"],
+    output: "datasets.csv"
+    run:
+        "calcium_imaging_automation/core/rules/setup.py"
+
+# import pandas as pd
 
-paths = pd.read_csv("datasets.csv")
+# paths = pd.read_csv("datasets.csv")
 
-rule all:
-    input:
-        expand("preprocess_output_{index}.txt", index=paths["index"])
+# rule all:
+#     input:
+#         expand("preprocess_output_{index}.txt", index=paths["index"])
 
-rule preprocess:
-    input:
-        lambda wildcards: paths.loc[int(wildcards.index), "read_dataset_path"],
-        lambda wildcards: paths.loc[int(wildcards.index), "write_dataset_path"],
-    output:
-        "preprocess_output_{index}.txt"
-    params:
-        index=lambda wildcards: wildcards.index
-    resources:
-        partition="fast",
-        mem_mb=16000,
-        cpu_per_task=1,
-        tasks=1,
-        nodes=1,
-    script:
-        "calcium_imaging_automation/core/rules/preprocess.py"
+# rule preprocess:
+#     input:
+#         lambda wildcards: paths.loc[int(wildcards.index), "read_dataset_path"],
+#         lambda wildcards: paths.loc[int(wildcards.index), "write_dataset_path"],
+#     output:
+#         "preprocess_output_{index}.txt"
+#     params:
+#         index=lambda wildcards: wildcards.index
+#     resources:
+#         partition="fast",
+#         mem_mb=16000,
+#         cpu_per_task=1,
+#         tasks=1,
+#         nodes=1,
+#     script:
+#         "calcium_imaging_automation/core/rules/preprocess.py"