From 76ff8b7f68a2ca96d4175b164abc6a6308ea4056 Mon Sep 17 00:00:00 2001 From: lauraporta Date: Wed, 4 Dec 2024 18:56:39 +0000 Subject: [PATCH] =?UTF-8?q?WIP:=20changing=20script=20=F0=9F=90=9B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- _datasets.csv | 21 +++++++ .../core/rules/preprocess.py | 2 - .../core/rules/setup.py | 63 ++++++------------- workflow/Snakefile | 56 +++++++++-------- 4 files changed, 69 insertions(+), 73 deletions(-) create mode 100644 _datasets.csv diff --git a/_datasets.csv b/_datasets.csv new file mode 100644 index 0000000..c9c0036 --- /dev/null +++ b/_datasets.csv @@ -0,0 +1,21 @@ +index,read_dataset_path,write_dataset_path +0,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230804_CAA_1119917,/ceph/margrie/laura/cimaut/derivatives/sub-0_230804_CAA_1119917 +1,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230818_CAA_1120210,/ceph/margrie/laura/cimaut/derivatives/sub-1_230818_CAA_1120210 +2,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230803_CAA_1119915,/ceph/margrie/laura/cimaut/derivatives/sub-2_230803_CAA_1119915 +3,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230801_CAA_1120181,/ceph/margrie/laura/cimaut/derivatives/sub-3_230801_CAA_1120181 +4,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230802_CAA_1120182,/ceph/margrie/laura/cimaut/derivatives/sub-4_230802_CAA_1120182 +5,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230803_CAA_1120181,/ceph/margrie/laura/cimaut/derivatives/sub-5_230803_CAA_1120181 +6,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230822_CAA_1120509,/ceph/margrie/laura/cimaut/derivatives/sub-6_230822_CAA_1120509 +7,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230823_CAA_1120181,/ceph/margrie/laura/cimaut/derivatives/sub-7_230823_CAA_1120181 +8,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230824_CAA_1119915,/ceph/margrie/laura/cimaut/derivatives/sub-8_230824_CAA_1119915 +9,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230825_CAA_1120182,/ceph/margrie/laura/cimaut/derivatives/sub-9_230825_CAA_1120182 +10,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230905_CAA_1119917,/ceph/margrie/laura/cimaut/derivatives/sub-10_230905_CAA_1119917 +11,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230907_CAA_1120210,/ceph/margrie/laura/cimaut/derivatives/sub-11_230907_CAA_1120210 +12,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230907_CAA_1120509,/ceph/margrie/laura/cimaut/derivatives/sub-12_230907_CAA_1120509 +13,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230912_CAA_1119915,/ceph/margrie/laura/cimaut/derivatives/sub-13_230912_CAA_1119915 +14,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230912_CAA_1120051,/ceph/margrie/laura/cimaut/derivatives/sub-14_230912_CAA_1120051 +15,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230913_CAA_1120182,/ceph/margrie/laura/cimaut/derivatives/sub-15_230913_CAA_1120182 +16,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230913_CAA_1120395,/ceph/margrie/laura/cimaut/derivatives/sub-16_230913_CAA_1120395 +17,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230914_CAA_1120181,/ceph/margrie/laura/cimaut/derivatives/sub-17_230914_CAA_1120181 +18,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230914_CAA_1120210,/ceph/margrie/laura/cimaut/derivatives/sub-18_230914_CAA_1120210 +19,/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/230915_CAA_1120509,/ceph/margrie/laura/cimaut/derivatives/sub-19_230915_CAA_1120509 diff --git a/calcium_imaging_automation/core/rules/preprocess.py b/calcium_imaging_automation/core/rules/preprocess.py index d45d7fd..e9dc3b1 100644 --- a/calcium_imaging_automation/core/rules/preprocess.py +++ b/calcium_imaging_automation/core/rules/preprocess.py @@ -10,8 +10,6 @@ write_dataset_path = Path(snakemake.input[1]) output = snakemake.output[0] - # os.system("module load miniconda") - # os.system("source activate /nfs/nhome/live/lporta/.conda/envs/cimat") output_path_dataset = write_dataset_path / "ses-0/funcimg/" data = derotate(read_dataset_path, output_path_dataset) diff --git a/calcium_imaging_automation/core/rules/setup.py b/calcium_imaging_automation/core/rules/setup.py index 98a88da..56f1835 100644 --- a/calcium_imaging_automation/core/rules/setup.py +++ b/calcium_imaging_automation/core/rules/setup.py @@ -6,19 +6,27 @@ from calcium_imaging_automation.core.reader import ReadAquiredData from calcium_imaging_automation.core.writer import DatashuttleWrapper +from snakemake.script import snakemake -def setup(raw_data_path, folder_read_pattern, file_read_pattern, output_path): +try: + read_dataset_path = Path(snakemake.input[0]) + write_dataset_path = Path(snakemake.input[1]) + folder_read_pattern = snakemake.params.folder_read_pattern + file_read_pattern = snakemake.params.file_read_pattern + + output = snakemake.output[0] + try: shutil.rmtree("/ceph/margrie/laura/cimaut/derivatives/") shutil.rmtree("/ceph/margrie/laura/cimaut/submitit/") except FileNotFoundError: print("No derivatives folder found") - print(f"Reading data from {raw_data_path}") + print(f"Reading data from {read_dataset_path}") reader = ReadAquiredData( - raw_data_path, + read_dataset_path, folder_read_pattern, file_read_pattern, ) @@ -27,7 +35,7 @@ def setup(raw_data_path, folder_read_pattern, file_read_pattern, output_path): number_of_tiffs = reader.max_session_number(filetype="tif") print(f"Max of tiffs found: {number_of_tiffs}") - writer = DatashuttleWrapper(output_path) + writer = DatashuttleWrapper(write_dataset_path) writer.create_folders(reader.dataset_names, session_number=number_of_tiffs) print("Folders created") @@ -40,44 +48,9 @@ def setup(raw_data_path, folder_read_pattern, file_read_pattern, output_path): ], } ) - datasets.to_csv("datasets.csv", index=True, index_label="index") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser( - description="Example usage of the pipeline manager." - ) - - parser.add_argument( - "raw_data_path", type=Path, help="Path to the raw data." - ) - parser.add_argument( - "output_path", type=Path, help="Path to the output data." - ) - parser.add_argument( - "--folder_read_pattern", - type=str, - help="Glob pattern for reading folder.", - default="*", - ) - parser.add_argument( - "--file_read_pattern", - type=str, - help="List of glob patterns for reading files.", - action="append", - ) - - args = parser.parse_args() - - try: - setup( - args.raw_data_path, - args.folder_read_pattern, - args.file_read_pattern, - args.output_path, - ) - - print("Success") - except Exception as e: - print(f"Error: {e.args}") - print(e.with_traceback(e.__traceback__)) + datasets.to_csv(output, index=True, index_label="index") + +except Exception as e: + print(e.args) + with open(output, "w") as f: + f.write(str(e.args)) diff --git a/workflow/Snakefile b/workflow/Snakefile index 19c6d7c..e817e52 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -1,31 +1,35 @@ rule setup: input: - datasets_path="/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/", - writing_path="/ceph/margrie/laura/cimaut/", - output: "setup_output.txt" - shell: "python calcium_imaging_automation/core/rules/setup.py {input.datasets_path} {input.writing_path} --folder_read_pattern '2*' --file_read_pattern 'rotation_00001.tif' --file_read_pattern '*.bin' > {output}" - -import pandas as pd + "/nfs/winstor/margrie/SimonWeiler/RawData/Invivo_imaging/3photon_rotation/shared/", + "/ceph/margrie/laura/cimaut/", + params: + folder_read_pattern="2*", + file_read_pattern=["rotation_00001.tif", "*.bin"], + output: "datasets.csv" + run: + "calcium_imaging_automation/core/rules/setup.py" + +# import pandas as pd -paths = pd.read_csv("datasets.csv") +# paths = pd.read_csv("datasets.csv") -rule all: - input: - expand("preprocess_output_{index}.txt", index=paths["index"]) +# rule all: +# input: +# expand("preprocess_output_{index}.txt", index=paths["index"]) -rule preprocess: - input: - lambda wildcards: paths.loc[int(wildcards.index), "read_dataset_path"], - lambda wildcards: paths.loc[int(wildcards.index), "write_dataset_path"], - output: - "preprocess_output_{index}.txt" - params: - index=lambda wildcards: wildcards.index - resources: - partition="fast", - mem_mb=16000, - cpu_per_task=1, - tasks=1, - nodes=1, - script: - "calcium_imaging_automation/core/rules/preprocess.py" +# rule preprocess: +# input: +# lambda wildcards: paths.loc[int(wildcards.index), "read_dataset_path"], +# lambda wildcards: paths.loc[int(wildcards.index), "write_dataset_path"], +# output: +# "preprocess_output_{index}.txt" +# params: +# index=lambda wildcards: wildcards.index +# resources: +# partition="fast", +# mem_mb=16000, +# cpu_per_task=1, +# tasks=1, +# nodes=1, +# script: +# "calcium_imaging_automation/core/rules/preprocess.py"