From aaa2dd07e65bde151f8c581bcfff65366d0b24f5 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Fri, 22 Mar 2024 16:58:05 +0100 Subject: [PATCH 01/86] [ENH]: DICOM to NIFTI BIDS conversion with heuristic generated from configuration.json file. Configuration file was slightly modified to handle dcm2niix options following nipype DC2NIIX attributes --- s2b_example_config_EMISEP_long.json | 4 +- shanoir2bidsv1.py | 672 ++++++++++++++++++++++++++++ 2 files changed, 674 insertions(+), 2 deletions(-) create mode 100755 shanoir2bidsv1.py diff --git a/s2b_example_config_EMISEP_long.json b/s2b_example_config_EMISEP_long.json index c024c5e..cf6f037 100644 --- a/s2b_example_config_EMISEP_long.json +++ b/s2b_example_config_EMISEP_long.json @@ -63,8 +63,8 @@ {"datasetName": "T1 MPRAGE CERV", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"} ], - "dcm2niix":"/home/mgaubert/Software/dcm2niix/dcm2niix_v1.0.20211006/dcm2niix", - "dcm2niix_options": "-v 0 -z y -ba n" + "dcm2niix":"~/softs/miniconda3/bin/dcm2niix", + "dcm2niix_options": {"verbose": true,"compress": "y", "anon_bids": false} } diff --git a/shanoir2bidsv1.py b/shanoir2bidsv1.py new file mode 100755 index 0000000..7016425 --- /dev/null +++ b/shanoir2bidsv1.py @@ -0,0 +1,672 @@ +#!/usr/bin/env python3 +DESCRIPTION = """ +shanoir2bids.py is a script that allows to download a Shanoir dataset and organise it as a BIDS data structure. + The script is made to run for every project given some information provided by the user into a ".json" + configuration file. More details regarding the configuration file in the Readme.md""" +# Script to download and BIDS-like organize data on Shanoir using "shanoir_downloader.py" developed by Arthur Masson +# @Author: Malo Gaubert , Quentin Duché +# @Date: 24 Juin 2022 +import os +import sys +import zipfile +import json +import shutil +import shanoir_downloader +from dotenv import load_dotenv +from pathlib import Path +from os.path import join as opj, splitext as ops, exists as ope, dirname as opd +from glob import glob +from time import time +import datetime +from dateutil import parser + +from heudiconv.main import workflow + + +# Load environment variables +load_dotenv(dotenv_path=opj(opd(__file__), ".env")) + + +def banner_msg(msg): + """ + Print a message framed by a banner of "*" characters + :param msg: + """ + banner = "*" * (len(msg) + 6) + print(banner + "\n* ", msg, " *\n" + banner) + + +# Keys for json configuration file +K_JSON_STUDY_NAME = "study_name" +K_JSON_L_SUBJECTS = "subjects" +K_JSON_SESSION = "session" +K_JSON_DATA_DICT = "data_to_bids" +K_JSON_FIND_AND_REPLACE = "find_and_replace_subject" +K_DCM2NIIX_PATH = "dcm2niix" +K_DCM2NIIX_OPTS = "dcm2niix_options" +K_FIND = "find" +K_REPLACE = "replace" +K_JSON_DATE_FROM = ( + "date_from" # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] +) +K_JSON_DATE_TO = ( + "date_to" # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] +) +LIST_MANDATORY_KEYS_JSON = [K_JSON_STUDY_NAME, K_JSON_L_SUBJECTS, K_JSON_DATA_DICT] +LIST_AUTHORIZED_KEYS_JSON = LIST_MANDATORY_KEYS_JSON + [ + K_DCM2NIIX_PATH, + K_DCM2NIIX_OPTS, + K_JSON_DATE_FROM, + K_JSON_DATE_TO, + K_JSON_SESSION, +] + +# Define keys for data dictionary +K_BIDS_NAME = "bidsName" +K_BIDS_DIR = "bidsDir" +K_BIDS_SES = "bidsSession" +K_DS_NAME = "datasetName" + +# Define Extensions that are dealt so far by (#todo : think of other possible extensions ?) +NIFTI = ".nii" +NIIGZ = ".nii.gz" +JSON = ".json" +BVAL = ".bval" +BVEC = ".bvec" +DCM = ".dcm" + +# Shanoir parameters +SHANOIR_FILE_TYPE_NIFTI = "nifti" +SHANOIR_FILE_TYPE_DICOM = "dicom" +DEFAULT_SHANOIR_FILE_TYPE = SHANOIR_FILE_TYPE_NIFTI + +# Define error and warning messages when call to dcm2niix is not well configured in the json file +DCM2NIIX_ERR_MSG = """ERROR !! +Conversion from DICOM to nifti can not be performed. +Please provide path to your favorite dcm2niix version in your Shanoir2BIDS .json configuration file. +Add key "{key}" with the absolute path to dcm2niix version to the following file : """ +DCM2NIIX_WARN_MSG = """WARNING. You did not provide any option to the dcm2niix call. +If you want to do so, add key "{key}" to you Shanoir2BIDS configuration file :""" + + +def check_date_format(date_to_format): + # TRUE FORMAT should be: date_format = 'Y-m-dTH:M:SZ' + try: + parser.parse(date_to_format) + # If the date validation goes wrong + except ValueError: + print( + "Incorrect data format, should be YYYY-MM-DDTHH:MM:SSZ (for example: 2020-02-19T00:00:00Z)" + ) + + +def read_json_config_file(json_file): + """ + Reads a json configuration file and checks whether mandatory keys for specifying the transformation from a + Shanoir dataset to a BIDS dataset is present. + :param json_file: str, path to a json configuration file + :return: + """ + f = open(json_file) + data = json.load(f) + # Check keys + for key in data.keys(): + if not key in LIST_AUTHORIZED_KEYS_JSON: + print('Unknown key "{}" for data dictionary'.format(key)) + for key in LIST_MANDATORY_KEYS_JSON: + if not key in data.keys(): + sys.exit('Error, missing key "{}" in data dictionary'.format(key)) + + # Sets the mandatory fields for the instance of the class + study_id = data[K_JSON_STUDY_NAME] + subjects = data[K_JSON_L_SUBJECTS] + data_dict = data[K_JSON_DATA_DICT] + + # Default non-mandatory options + list_fars = [] + dcm2niix_path = None + dcm2niix_opts = None + date_from = "*" + date_to = "*" + session_id = "*" + + if K_JSON_FIND_AND_REPLACE in data.keys(): + list_fars = data[K_JSON_FIND_AND_REPLACE] + if K_DCM2NIIX_PATH in data.keys(): + dcm2niix_path = data[K_DCM2NIIX_PATH] + if K_DCM2NIIX_OPTS in data.keys(): + dcm2niix_opts = data[K_DCM2NIIX_OPTS] + if K_JSON_DATE_FROM in data.keys(): + if data[K_JSON_DATE_FROM] == "": + data_from = "*" + else: + date_from = data[K_JSON_DATE_FROM] + check_date_format(date_from) + if K_JSON_DATE_TO in data.keys(): + if data[K_JSON_DATE_TO] == "": + data_to = "*" + else: + date_to = data[K_JSON_DATE_TO] + check_date_format(date_to) + if K_JSON_SESSION in data.keys(): + session_id = data[K_JSON_SESSION] + + # Close json file and return + f.close() + return ( + study_id, + subjects, + session_id, + data_dict, + list_fars, + dcm2niix_path, + dcm2niix_opts, + date_from, + date_to, + ) + + +def generate_heuristic_file( + shanoir2bids_dict: object, path_heuristic_file: object +) -> None: + """Generate heudiconv heuristic.py file from shanoir2bids mapping dict + Parameters + ---------- + shanoir2bids_dict : + path_heuristic_file : path of the python heuristic file (.py) + """ + heuristic = f"""from heudiconv.heuristics.reproin import create_key + +def create_bids_key(dataset): + template = create_key(subdir=dataset['bidsDir'],file_suffix=dataset['bidsName'],outtype=("dicom","nii.gz")) + return template + +def get_dataset_to_key_mapping(shanoir2bids): + dataset_to_key = dict() + for dataset in shanoir2bids: + template = create_bids_key(dataset) + dataset_to_key[dataset['datasetName']] = template + return dataset_to_key + + +def infotodict(seqinfo): + + info = dict() + shanoir2bids = {shanoir2bids_dict} + + dataset_to_key = get_dataset_to_key_mapping(shanoir2bids) + for seq in seqinfo: + if seq.series_description in dataset_to_key.keys(): + key = dataset_to_key[seq.series_description] + if key in info.keys(): + info[key].append(seq.series_id) + else: + info[key] = [seq.series_id] + return info +""" + + with open(path_heuristic_file, "w", encoding="utf-8") as file: + file.write(heuristic) + file.close() + pass + + +class DownloadShanoirDatasetToBIDS: + """ + class that handles the downloading of shanoir data set and the reformatting as a BIDS data structure + """ + + def __init__(self): + """ + Initialize the class instance + """ + self.shanoir_subjects = None # List of Shanoir subjects + self.shanoir2bids_dict = ( + None # Dictionary specifying how to reformat data into BIDS structure + ) + self.shanoir_username = None # Shanoir username + self.shanoir_study_id = None # Shanoir study ID + self.shanoir_session_id = None # Shanoir study ID + self.shanoir_file_type = ( + DEFAULT_SHANOIR_FILE_TYPE # Default download type (nifti/dicom) + ) + self.json_config_file = None + self.list_fars = [] # List of substrings to edit in subjects names + self.dl_dir = None # download directory, where data will be stored + self.parser = None # Shanoir Downloader Parser + self.n_seq = 0 # Number of sequences in the shanoir2bids_dict + self.log_fn = None + self.dcm2niix_path = None # Path to the dcm2niix the user wants to use + self.dcm2niix_opts = None # Options to add to the dcm2niix call + self.dcm2niix_config_file = None # .json file to store dcm2niix options + self.date_from = None + self.date_to = None + self.longitudinal = False + self.to_automri_format = ( + False # Special filenames for automri (close to BIDS format) + ) + self.add_sns = False # Add series number suffix to filename + + def set_json_config_file(self, json_file): + """ + Sets the configuration for the download through a json file + :param json_file: str, path to the json_file + """ + self.json_config_file = json_file + ( + study_id, + subjects, + session_id, + data_dict, + list_fars, + dcm2niix_path, + dcm2niix_opts, + date_from, + date_to, + ) = read_json_config_file(json_file=json_file) + self.set_shanoir_study_id(study_id=study_id) + self.set_shanoir_subjects(subjects=subjects) + self.set_shanoir_session_id(session_id=session_id) + self.set_shanoir2bids_dict(data_dict=data_dict) + self.set_shanoir_list_find_and_replace(list_fars=list_fars) + self.set_dcm2niix_parameters( + dcm2niix_path=dcm2niix_path, dcm2niix_opts=dcm2niix_opts + ) + self.set_date_from(date_from=date_from) + self.set_date_to(date_to=date_to) + + def set_shanoir_file_type(self, shanoir_file_type): + if shanoir_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI]: + self.shanoir_file_type = shanoir_file_type + else: + sys.exit("Unknown shanoir file type {}".format(shanoir_file_type)) + + def set_shanoir_study_id(self, study_id): + self.shanoir_study_id = study_id + + def set_shanoir_username(self, shanoir_username): + self.shanoir_username = shanoir_username + + def set_shanoir_domaine(self, shanoir_domaine): + self.shanoir_domaine = shanoir_domaine + + def set_shanoir_subjects(self, subjects): + self.shanoir_subjects = subjects + + def set_shanoir_session_id(self, session_id): + self.shanoir_session_id = session_id + + def set_shanoir_list_find_and_replace(self, list_fars): + self.list_fars = list_fars + + def set_dcm2niix_parameters(self, dcm2niix_path, dcm2niix_opts): + self.dcm2niix_path = dcm2niix_path + self.dcm2niix_opts = dcm2niix_opts + + def set_dcm2niix_config_files(self, path_dcm2niix_options_files): + self.dcm2niix_config_file = path_dcm2niix_options_files + # Serializing json + json_object = json.dumps(self.dcm2niix_opts, indent=4) + with open(self.dcm2niix_config_file, "w") as file: + file.write(json_object) + + def set_date_from(self, date_from): + self.date_from = date_from + + def set_date_to(self, date_to): + self.date_to = date_to + + def set_shanoir2bids_dict(self, data_dict): + self.shanoir2bids_dict = data_dict + self.n_seq = len(self.shanoir2bids_dict) + + def set_download_directory(self, dl_dir): + if dl_dir is None: + # Create a default download directory + dt = datetime.datetime.now().strftime("%Y_%m_%d_at_%Hh%Mm%Ss") + self.dl_dir = "_".join( + ["shanoir2bids", "download", self.shanoir_study_id, dt] + ) + print( + "A NEW DEFAULT directory is created as you did not provide a download directory (-of option)\n\t" + + self.dl_dir + ) + else: + self.dl_dir = dl_dir + # Create directory if it does not exist + if not ope(self.dl_dir): + Path(self.dl_dir).mkdir(parents=True, exist_ok=True) + self.set_log_filename() + + def set_heuristic_file(self, path_heuristic_file): + if path_heuristic_file is None: + print("TO BE DONE") + else: + self.heuristic_file = path_heuristic_file + + def set_log_filename(self): + curr_time = datetime.datetime.now() + basename = "shanoir_downloader_{:04}{:02}{:02}_{:02}{:02}{:02}.log".format( + curr_time.year, + curr_time.month, + curr_time.day, + curr_time.hour, + curr_time.minute, + curr_time.second, + ) + self.log_fn = opj(self.dl_dir, basename) + + def toggle_longitudinal_version(self): + self.longitudinal = True + + def switch_to_automri_format(self): + self.to_automri_format = True + + def add_series_number_suffix(self): + self.add_sns = True + + def configure_parser(self): + """ + Configure the parser and the configuration of the shanoir_downloader + """ + self.parser = shanoir_downloader.create_arg_parser() + shanoir_downloader.add_common_arguments(self.parser) + shanoir_downloader.add_configuration_arguments(self.parser) + shanoir_downloader.add_search_arguments(self.parser) + shanoir_downloader.add_ids_arguments(self.parser) + + def download_subject(self, subject_to_search): + """ + For a single subject + 1. Downloads the Shanoir datasets + 2. Reorganises the Shanoir dataset as BIDS format as defined in the json configuration file provided by user + :param subject_to_search: + :return: + """ + banner_msg("Downloading subject " + subject_to_search) + + # Open log file to write the steps of processing (downloading, renaming...) + fp = open(self.log_fn, "a") + + # Real Shanoir2Bids mapping ( deal with search terms are included in datasetName field) + bids_mapping = [] + + # Loop on each sequence defined in the dictionary + for seq in range(self.n_seq): + # Isolate elements that are called many times + shanoir_seq_name = self.shanoir2bids_dict[seq][ + K_DS_NAME + ] # Shanoir sequence name (OLD) + bids_seq_subdir = self.shanoir2bids_dict[seq][ + K_BIDS_DIR + ] # Sequence BIDS subdirectory name (NEW) + bids_seq_name = self.shanoir2bids_dict[seq][ + K_BIDS_NAME + ] # Sequence BIDS nickname (NEW) + if self.longitudinal: + bids_seq_session = self.shanoir2bids_dict[seq][ + K_BIDS_SES + ] # Sequence BIDS nickname (NEW) + + # Print message concerning the sequence that is being downloaded + print( + "\t-", + bids_seq_name, + subject_to_search, + "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", + ) + + # Initialize the parser + search_txt = ( + "studyName:" + + self.shanoir_study_id.replace(" ", "?") + + " AND datasetName:" + + shanoir_seq_name.replace(" ", "?") + + " AND subjectName:" + + subject_to_search.replace(" ", "?") + + " AND examinationComment:" + + self.shanoir_session_id.replace(" ", "*") + + " AND examinationDate:[" + + self.date_from + + " TO " + + self.date_to + + "]" + ) + + args = self.parser.parse_args( + [ + "-u", + self.shanoir_username, + "-d", + self.shanoir_domaine, + "-of", + self.dl_dir, + "-em", + "-st", + search_txt, + "-s", + "200", + "-f", + self.shanoir_file_type, + "-so", + "id,ASC", + "-t", + "500", + ] + ) # Increase time out for heavy files + + config = shanoir_downloader.initialize(args) + response = shanoir_downloader.solr_search(config, args) + + # From response, process the data + # Print the number of items found and a list of these items + if response.status_code == 200: + # Invoke shanoir_downloader to download all the data + shanoir_downloader.download_search_results(config, args, response) + + if len(response.json()["content"]) == 0: + warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns +a result on the website. +Search Text : "{}" \n""".format( + search_txt + ) + print(warn_msg) + fp.write(warn_msg) + else: + for item in response.json()["content"]: + # Define subject_id + su_id = item["subjectName"] + # If the user has defined a list of edits to subject names... then do the find and replace + for far in self.list_fars: + su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) + + # ID of the subject (sub-*) + subject_id = su_id + # correct BIDS mapping of the searched dataset + bids_seq_mapping = { + "datasetName": item["datasetName"], + "bidsDir": bids_seq_subdir, + "bidsName": bids_seq_name, + "bids_subject_id": subject_id, + } + + if self.longitudinal: + bids_seq_mapping["bids_session_id"] = bids_seq_session + else: + bids_seq_mapping["bids_session_id"] = None + + bids_mapping.append(bids_seq_mapping) + + # Write the information on the data in the log file + fp.write("- datasetId = " + str(item["datasetId"]) + "\n") + fp.write(" -- studyName: " + item["studyName"] + "\n") + fp.write(" -- subjectName: " + item["subjectName"] + "\n") + fp.write(" -- session: " + item["examinationComment"] + "\n") + fp.write(" -- datasetName: " + item["datasetName"] + "\n") + fp.write( + " -- examinationDate: " + item["examinationDate"] + "\n" + ) + fp.write(" >> Downloading archive OK\n") + + # Create temp directory to make sure the directory is empty before + # TODO: Replace with temp directory ? + tmp_dir = opj(self.dl_dir, "temp_archive") + Path(tmp_dir).mkdir(parents=True, exist_ok=True) + + # Extract the downloaded archive + dl_archive = glob(opj(self.dl_dir, "*" + item["id"] + "*.zip"))[ + 0 + ] + with zipfile.ZipFile(dl_archive, "r") as zip_ref: + extraction_dir = opj(tmp_dir, item["id"]) + zip_ref.extractall(extraction_dir) + + fp.write( + " >> Extraction of all files from archive '" + + dl_archive + + " into " + + tmp_dir + + item["id"] + + "\n" + ) + + elif response.status_code == 204: + banner_msg("ERROR : No file found!") + fp.write(" >> ERROR : No file found!\n") + else: + banner_msg( + "ERROR : Returned by the request: status of the response = " + + response.status_code + ) + fp.write( + " >> ERROR : Returned by the request: status of the response = " + + str(response.status_code) + + "\n" + ) + # Generate Heudiconv heuristic file from .json mapping + generate_heuristic_file(bids_mapping, self.heuristic_file) + # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options + + if self.longitudinal: + workflow( + files=glob(opj(self.dl_dir, 'temp_archive', "*", "*.dcm"), recursive=True), + outdir=opj(self.dl_dir, "test"), + subjs=[subject_id], + session = bids_seq_session, + converter="dcm2niix", + heuristic=self.heuristic_file, + bids_options='--bids', + dcmconfig=self.dcm2niix_config_file, + datalad=True, + minmeta=True, + ) + else: + + workflow( + files=glob(opj(self.dl_dir,'temp_archive',"*", "*.dcm"), recursive=True), + outdir=opj(self.dl_dir, "test"), + subjs=[subject_id], + converter="dcm2niix", + heuristic=self.heuristic_file, + bids_options='--bids', + dcmconfig=self.dcm2niix_config_file, + datalad=True, + minmeta=True, + ) + fp.close() + + def download(self): + """ + Loop over the Shanoir subjects and go download the required datasets + :return: + """ + self.set_log_filename() + self.configure_parser() # Configure the shanoir_downloader parser + fp = open(self.log_fn, "w") + for subject_to_search in self.shanoir_subjects: + t_start_subject = time() + self.download_subject(subject_to_search=subject_to_search) + dur_min = int((time() - t_start_subject) // 60) + dur_sec = int((time() - t_start_subject) % 60) + end_msg = ( + "Downloaded dataset for subject " + + subject_to_search + + " in {}m{}s".format(dur_min, dur_sec) + ) + banner_msg(end_msg) + + +def main(): + # Parse argument for the script + parser = shanoir_downloader.create_arg_parser(description=DESCRIPTION) + # Use username and output folder arguments from shanoir_downloader + shanoir_downloader.add_username_argument(parser) + parser.add_argument( + "-d", + "--domain", + default="shanoir.irisa.fr", + help="The shanoir domain to query.", + ) + parser.add_argument( + "-f", + "--format", + default="dicom", + choices=["dicom"], + help="The format to download.", + ) + shanoir_downloader.add_output_folder_argument(parser=parser, required=False) + # Add the argument for the configuration file + parser.add_argument( + "-j", + "--config_file", + required=True, + help="Path to the .json configuration file specifying parameters for shanoir downloading.", + ) + parser.add_argument( + "-L", + "--longitudinal", + required=False, + action="store_true", + help="Toggle longitudinal approach.", + ) + # parser.add_argument( + # "-a", "--automri", action="store_true", help="Switch to automri file tree." + # ) + # parser.add_argument( + # "-A", + # "--add_sns", + # action="store_true", + # help="Add series number suffix (compatible with -a)", + # ) + # Parse arguments + args = parser.parse_args() + + # Start configuring the DownloadShanoirDatasetToBids class instance + stb = DownloadShanoirDatasetToBIDS() + stb.set_shanoir_username(args.username) + stb.set_shanoir_domaine(args.domain) + stb.set_json_config_file( + json_file=args.config_file + ) # path to json configuration file + stb.set_shanoir_file_type(shanoir_file_type=args.format) # Format (dicom or nifti) + stb.set_download_directory( + dl_dir=args.output_folder + ) # output folder (if None a default directory is created) + stb.set_heuristic_file(path_heuristic_file="/home/alpron/heuristic.py") + stb.set_dcm2niix_config_files( + path_dcm2niix_options_files="/home/alpron/dcm2niix_options.json" + ) + if args.longitudinal: + stb.toggle_longitudinal_version() + # if args.automri: + # stb.switch_to_automri_format() + # if args.add_sns: + # if not args.automri: + # print("Warning : -A option is only compatible with -a option.") + # stb.add_series_number_suffix() + + stb.download() + + +if __name__ == "__main__": + main() From 35a8d71bc44ec8c6df4d69185858d99391528f29 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 25 Mar 2024 11:48:34 +0100 Subject: [PATCH 02/86] - added temporary directories to download in DICOM archives and extracted and heuristic and tempfiles for dcm2niix configuration and heudivconv heuristic files. --- shanoir2bidsv1.py | 401 ++++++++++++++++++++++++---------------------- 1 file changed, 210 insertions(+), 191 deletions(-) diff --git a/shanoir2bidsv1.py b/shanoir2bidsv1.py index 7016425..e302238 100755 --- a/shanoir2bidsv1.py +++ b/shanoir2bidsv1.py @@ -6,20 +6,22 @@ # Script to download and BIDS-like organize data on Shanoir using "shanoir_downloader.py" developed by Arthur Masson # @Author: Malo Gaubert , Quentin Duché # @Date: 24 Juin 2022 + import os -import sys -import zipfile -import json -import shutil -import shanoir_downloader -from dotenv import load_dotenv -from pathlib import Path from os.path import join as opj, splitext as ops, exists as ope, dirname as opd from glob import glob +import sys +from pathlib import Path from time import time +import zipfile import datetime +import tempfile from dateutil import parser +import json +import shutil +import shanoir_downloader +from dotenv import load_dotenv from heudiconv.main import workflow @@ -238,7 +240,6 @@ def __init__(self): self.log_fn = None self.dcm2niix_path = None # Path to the dcm2niix the user wants to use self.dcm2niix_opts = None # Options to add to the dcm2niix call - self.dcm2niix_config_file = None # .json file to store dcm2niix options self.date_from = None self.date_to = None self.longitudinal = False @@ -303,11 +304,10 @@ def set_dcm2niix_parameters(self, dcm2niix_path, dcm2niix_opts): self.dcm2niix_path = dcm2niix_path self.dcm2niix_opts = dcm2niix_opts - def set_dcm2niix_config_files(self, path_dcm2niix_options_files): - self.dcm2niix_config_file = path_dcm2niix_options_files + def export_dcm2niix_config_options(self, path_dcm2niix_options_file): # Serializing json json_object = json.dumps(self.dcm2niix_opts, indent=4) - with open(self.dcm2niix_config_file, "w") as file: + with open(path_dcm2niix_options_file, "w") as file: file.write(json_object) def set_date_from(self, date_from): @@ -340,9 +340,15 @@ def set_download_directory(self, dl_dir): def set_heuristic_file(self, path_heuristic_file): if path_heuristic_file is None: - print("TO BE DONE") + print(f"No heuristic file provided") else: - self.heuristic_file = path_heuristic_file + filename, ext = ops(path_heuristic_file) + if ext != ".py": + print( + f"Provided heuristic file {path_heuristic_file} is not a .py file as expected" + ) + else: + self.heuristic_file = path_heuristic_file def set_log_filename(self): curr_time = datetime.datetime.now() @@ -388,192 +394,205 @@ def download_subject(self, subject_to_search): # Open log file to write the steps of processing (downloading, renaming...) fp = open(self.log_fn, "a") - # Real Shanoir2Bids mapping ( deal with search terms are included in datasetName field) + # Real Shanoir2Bids mapping (handle case when solr search term are included) bids_mapping = [] - # Loop on each sequence defined in the dictionary - for seq in range(self.n_seq): - # Isolate elements that are called many times - shanoir_seq_name = self.shanoir2bids_dict[seq][ - K_DS_NAME - ] # Shanoir sequence name (OLD) - bids_seq_subdir = self.shanoir2bids_dict[seq][ - K_BIDS_DIR - ] # Sequence BIDS subdirectory name (NEW) - bids_seq_name = self.shanoir2bids_dict[seq][ - K_BIDS_NAME - ] # Sequence BIDS nickname (NEW) - if self.longitudinal: - bids_seq_session = self.shanoir2bids_dict[seq][ - K_BIDS_SES - ] # Sequence BIDS nickname (NEW) - - # Print message concerning the sequence that is being downloaded - print( - "\t-", - bids_seq_name, - subject_to_search, - "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", - ) - - # Initialize the parser - search_txt = ( - "studyName:" - + self.shanoir_study_id.replace(" ", "?") - + " AND datasetName:" - + shanoir_seq_name.replace(" ", "?") - + " AND subjectName:" - + subject_to_search.replace(" ", "?") - + " AND examinationComment:" - + self.shanoir_session_id.replace(" ", "*") - + " AND examinationDate:[" - + self.date_from - + " TO " - + self.date_to - + "]" - ) - - args = self.parser.parse_args( - [ - "-u", - self.shanoir_username, - "-d", - self.shanoir_domaine, - "-of", - self.dl_dir, - "-em", - "-st", - search_txt, - "-s", - "200", - "-f", - self.shanoir_file_type, - "-so", - "id,ASC", - "-t", - "500", - ] - ) # Increase time out for heavy files - - config = shanoir_downloader.initialize(args) - response = shanoir_downloader.solr_search(config, args) - - # From response, process the data - # Print the number of items found and a list of these items - if response.status_code == 200: - # Invoke shanoir_downloader to download all the data - shanoir_downloader.download_search_results(config, args, response) - - if len(response.json()["content"]) == 0: - warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns -a result on the website. -Search Text : "{}" \n""".format( - search_txt + # temporary directory containing dowloaded DICOM.zip files + with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_dicom: + with tempfile.TemporaryDirectory( + dir=self.dl_dir + ) as tmp_archive: + print(tmp_archive) + # Loop on each sequence defined in the dictionary + for seq in range(self.n_seq): + # Isolate elements that are called many times + shanoir_seq_name = self.shanoir2bids_dict[seq][ + K_DS_NAME + ] # Shanoir sequence name (OLD) + bids_seq_subdir = self.shanoir2bids_dict[seq][ + K_BIDS_DIR + ] # Sequence BIDS subdirectory name (NEW) + bids_seq_name = self.shanoir2bids_dict[seq][ + K_BIDS_NAME + ] # Sequence BIDS nickname (NEW) + if self.longitudinal: + bids_seq_session = self.shanoir2bids_dict[seq][ + K_BIDS_SES + ] # Sequence BIDS nickname (NEW) + + # Print message concerning the sequence that is being downloaded + print( + "\t-", + bids_seq_name, + subject_to_search, + "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", ) - print(warn_msg) - fp.write(warn_msg) - else: - for item in response.json()["content"]: - # Define subject_id - su_id = item["subjectName"] - # If the user has defined a list of edits to subject names... then do the find and replace - for far in self.list_fars: - su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) - - # ID of the subject (sub-*) - subject_id = su_id - # correct BIDS mapping of the searched dataset - bids_seq_mapping = { - "datasetName": item["datasetName"], - "bidsDir": bids_seq_subdir, - "bidsName": bids_seq_name, - "bids_subject_id": subject_id, - } - - if self.longitudinal: - bids_seq_mapping["bids_session_id"] = bids_seq_session - else: - bids_seq_mapping["bids_session_id"] = None - bids_mapping.append(bids_seq_mapping) + # Initialize the parser + search_txt = ( + "studyName:" + + self.shanoir_study_id.replace(" ", "?") + + " AND datasetName:" + + shanoir_seq_name.replace(" ", "?") + + " AND subjectName:" + + subject_to_search.replace(" ", "?") + + " AND examinationComment:" + + self.shanoir_session_id.replace(" ", "*") + + " AND examinationDate:[" + + self.date_from + + " TO " + + self.date_to + + "]" + ) - # Write the information on the data in the log file - fp.write("- datasetId = " + str(item["datasetId"]) + "\n") - fp.write(" -- studyName: " + item["studyName"] + "\n") - fp.write(" -- subjectName: " + item["subjectName"] + "\n") - fp.write(" -- session: " + item["examinationComment"] + "\n") - fp.write(" -- datasetName: " + item["datasetName"] + "\n") - fp.write( - " -- examinationDate: " + item["examinationDate"] + "\n" - ) - fp.write(" >> Downloading archive OK\n") + args = self.parser.parse_args( + [ + "-u", + self.shanoir_username, + "-d", + self.shanoir_domaine, + "-of", + tmp_archive, + "-em", + "-st", + search_txt, + "-s", + "200", + "-f", + self.shanoir_file_type, + "-so", + "id,ASC", + "-t", + "500", + ] + ) # Increase time out for heavy files - # Create temp directory to make sure the directory is empty before - # TODO: Replace with temp directory ? - tmp_dir = opj(self.dl_dir, "temp_archive") - Path(tmp_dir).mkdir(parents=True, exist_ok=True) + config = shanoir_downloader.initialize(args) + response = shanoir_downloader.solr_search(config, args) - # Extract the downloaded archive - dl_archive = glob(opj(self.dl_dir, "*" + item["id"] + "*.zip"))[ - 0 - ] - with zipfile.ZipFile(dl_archive, "r") as zip_ref: - extraction_dir = opj(tmp_dir, item["id"]) - zip_ref.extractall(extraction_dir) + # From response, process the data + # Print the number of items found and a list of these items + if response.status_code == 200: + # Invoke shanoir_downloader to download all the data + shanoir_downloader.download_search_results( + config, args, response + ) + if len(response.json()["content"]) == 0: + warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns + a result on the website. + Search Text : "{}" \n""".format( + search_txt + ) + print(warn_msg) + fp.write(warn_msg) + else: + for item in response.json()["content"]: + # Define subject_id + su_id = item["subjectName"] + # If the user has defined a list of edits to subject names... then do the find and replace + for far in self.list_fars: + su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) + + # ID of the subject (sub-*) + subject_id = su_id + # correct BIDS mapping of the searched dataset + bids_seq_mapping = { + "datasetName": item["datasetName"], + "bidsDir": bids_seq_subdir, + "bidsName": bids_seq_name, + "bids_subject_id": subject_id, + } + + if self.longitudinal: + bids_seq_mapping[ + "bids_session_id" + ] = bids_seq_session + else: + bids_seq_mapping["bids_session_id"] = None + + bids_mapping.append(bids_seq_mapping) + + # Write the information on the data in the log file + fp.write( + "- datasetId = " + str(item["datasetId"]) + "\n" + ) + fp.write(" -- studyName: " + item["studyName"] + "\n") + fp.write( + " -- subjectName: " + item["subjectName"] + "\n" + ) + fp.write( + " -- session: " + item["examinationComment"] + "\n" + ) + fp.write( + " -- datasetName: " + item["datasetName"] + "\n" + ) + fp.write( + " -- examinationDate: " + + item["examinationDate"] + + "\n" + ) + fp.write(" >> Downloading archive OK\n") + + # Extract the downloaded archive + dl_archive = glob( + opj(tmp_archive, "*" + item["id"] + "*.zip") + )[0] + with zipfile.ZipFile(dl_archive, "r") as zip_ref: + extraction_dir = opj(tmp_dicom, item["id"]) + zip_ref.extractall(extraction_dir) + + fp.write( + " >> Extraction of all files from archive '" + + dl_archive + + " into " + + extraction_dir + + "\n" + ) + + elif response.status_code == 204: + banner_msg("ERROR : No file found!") + fp.write(" >> ERROR : No file found!\n") + else: + banner_msg( + "ERROR : Returned by the request: status of the response = " + + response.status_code + ) fp.write( - " >> Extraction of all files from archive '" - + dl_archive - + " into " - + tmp_dir - + item["id"] + " >> ERROR : Returned by the request: status of the response = " + + str(response.status_code) + "\n" ) - elif response.status_code == 204: - banner_msg("ERROR : No file found!") - fp.write(" >> ERROR : No file found!\n") - else: - banner_msg( - "ERROR : Returned by the request: status of the response = " - + response.status_code - ) - fp.write( - " >> ERROR : Returned by the request: status of the response = " - + str(response.status_code) - + "\n" - ) - # Generate Heudiconv heuristic file from .json mapping - generate_heuristic_file(bids_mapping, self.heuristic_file) - # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options - - if self.longitudinal: - workflow( - files=glob(opj(self.dl_dir, 'temp_archive', "*", "*.dcm"), recursive=True), - outdir=opj(self.dl_dir, "test"), - subjs=[subject_id], - session = bids_seq_session, - converter="dcm2niix", - heuristic=self.heuristic_file, - bids_options='--bids', - dcmconfig=self.dcm2niix_config_file, - datalad=True, - minmeta=True, - ) - else: - - workflow( - files=glob(opj(self.dl_dir,'temp_archive',"*", "*.dcm"), recursive=True), - outdir=opj(self.dl_dir, "test"), - subjs=[subject_id], - converter="dcm2niix", - heuristic=self.heuristic_file, - bids_options='--bids', - dcmconfig=self.dcm2niix_config_file, - datalad=True, - minmeta=True, - ) - fp.close() + # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options + with tempfile.NamedTemporaryFile(mode='r+', + encoding="utf-8", dir=self.dl_dir, suffix=".py" + ) as heuristic_file: + # Generate Heudiconv heuristic file from configuration.json mapping + generate_heuristic_file(bids_mapping, heuristic_file.name) + with tempfile.NamedTemporaryFile(mode='r+', + encoding="utf-8", dir=self.dl_dir, suffix=".json" + ) as dcm2niix_config_file: + self.export_dcm2niix_config_options(dcm2niix_config_file.name) + workflow_params = { + "files": glob( + opj(tmp_dicom, "*", "*.dcm"), recursive=True + ), + "outdir": opj(self.dl_dir, str(self.shanoir_study_id)), + "subjs": [subject_id], + "converter": "dcm2niix", + "heuristic": heuristic_file.name, + "bids_options": "--bids", + "dcmconfig": dcm2niix_config_file.name, + "datalad": True, + "minmeta": True, + } + + if self.longitudinal: + workflow_params["session"] = bids_seq_session + + workflow(**workflow_params) + fp.close() def download(self): """ @@ -652,10 +671,10 @@ def main(): stb.set_download_directory( dl_dir=args.output_folder ) # output folder (if None a default directory is created) - stb.set_heuristic_file(path_heuristic_file="/home/alpron/heuristic.py") - stb.set_dcm2niix_config_files( - path_dcm2niix_options_files="/home/alpron/dcm2niix_options.json" - ) + # stb.set_heuristic_file(path_heuristic_file="/home/alpron/heuristic.py") + # stb.set_dcm2niix_config_files( + # path_dcm2niix_options_files="/home/alpron/dcm2niix_options.json" + # ) if args.longitudinal: stb.toggle_longitudinal_version() # if args.automri: From a66b273f173ecaffb3ca7a24622cc747d9f3606c Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 25 Mar 2024 14:11:16 +0100 Subject: [PATCH 03/86] - renamed few modality suffixes so that it fits BIDS spec --- s2b_example_config_EMISEP_long.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/s2b_example_config_EMISEP_long.json b/s2b_example_config_EMISEP_long.json index cf6f037..bc64ab7 100644 --- a/s2b_example_config_EMISEP_long.json +++ b/s2b_example_config_EMISEP_long.json @@ -4,8 +4,8 @@ "session": "08-74 MO ENCEPHALE", "data_to_bids": [ - {"datasetName": "3D T1 MPRAGE", "bidsDir": "brain", "bidsName": "t1w", "bidsSession": "M00"}, - {"datasetName": "*3d flair*", "bidsDir": "brain", "bidsName": "flair", "bidsSession": "M00"}, + {"datasetName": "3D T1 MPRAGE", "bidsDir": "brain", "bidsName": "T1w", "bidsSession": "M00"}, + {"datasetName": "*3d flair*", "bidsDir": "brain", "bidsName": "FLAIR", "bidsSession": "M00"}, {"datasetName": "*c1c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", "bidsSession": "M00"}, {"datasetName": "*c1-c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", "bidsSession": "M00"}, @@ -64,7 +64,7 @@ ], "dcm2niix":"~/softs/miniconda3/bin/dcm2niix", - "dcm2niix_options": {"verbose": true,"compress": "y", "anon_bids": false} + "dcm2niix_options": {"verbose": true,"compress": "y", "anon_bids": true} } From 0a12916f5f41bf0d4c604ec32a84b30c3d82dcb0 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 26 Mar 2024 10:39:15 +0100 Subject: [PATCH 04/86] change main script name --- shanoir2bidsv1.py => shanoir2bids_heudiconv.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename shanoir2bidsv1.py => shanoir2bids_heudiconv.py (100%) diff --git a/shanoir2bidsv1.py b/shanoir2bids_heudiconv.py similarity index 100% rename from shanoir2bidsv1.py rename to shanoir2bids_heudiconv.py From 7f78df14532e45280b6c46323fd268c6fe9b1ca6 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Wed, 27 Mar 2024 13:58:41 +0100 Subject: [PATCH 05/86] + modified default heudiconv StudyID grouping (used all) to handle multiple StudyID per Examination + included run simplifications --- shanoir2bids_heudiconv.py | 41 ++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index e302238..dcb5b55 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -18,11 +18,14 @@ import tempfile from dateutil import parser import json +import logging import shutil import shanoir_downloader from dotenv import load_dotenv from heudiconv.main import workflow +# import loggger used in heudiconv workflow +from heudiconv.main import lgr # Load environment variables @@ -180,7 +183,8 @@ def generate_heuristic_file( heuristic = f"""from heudiconv.heuristics.reproin import create_key def create_bids_key(dataset): - template = create_key(subdir=dataset['bidsDir'],file_suffix=dataset['bidsName'],outtype=("dicom","nii.gz")) + + template = create_key(subdir=dataset['bidsDir'],file_suffix=r"run-{{item:02d}}_" + dataset['bidsName'],outtype=("dicom","nii.gz")) return template def get_dataset_to_key_mapping(shanoir2bids): @@ -190,6 +194,16 @@ def get_dataset_to_key_mapping(shanoir2bids): dataset_to_key[dataset['datasetName']] = template return dataset_to_key +def simplify_runs(info): + info_final = dict() + for key in info.keys(): + if len(info[key])==1: + new_template = key[0].replace('run-{{item:02d}}_','') + new_key = (new_template, key[1], key[2]) + info_final[new_key] = info[key] + else: + info_final[key] = info[key] + return info_final def infotodict(seqinfo): @@ -204,7 +218,9 @@ def infotodict(seqinfo): info[key].append(seq.series_id) else: info[key] = [seq.series_id] - return info + # remove run- key if not needed (one run only) + info_final = simplify_runs(info) + return info_final """ with open(path_heuristic_file, "w", encoding="utf-8") as file: @@ -396,12 +412,9 @@ def download_subject(self, subject_to_search): # Real Shanoir2Bids mapping (handle case when solr search term are included) bids_mapping = [] - # temporary directory containing dowloaded DICOM.zip files with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_dicom: - with tempfile.TemporaryDirectory( - dir=self.dl_dir - ) as tmp_archive: + with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_archive: print(tmp_archive) # Loop on each sequence defined in the dictionary for seq in range(self.n_seq): @@ -565,33 +578,35 @@ def download_subject(self, subject_to_search): ) # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options - with tempfile.NamedTemporaryFile(mode='r+', - encoding="utf-8", dir=self.dl_dir, suffix=".py" + with tempfile.NamedTemporaryFile( + mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" ) as heuristic_file: # Generate Heudiconv heuristic file from configuration.json mapping generate_heuristic_file(bids_mapping, heuristic_file.name) - with tempfile.NamedTemporaryFile(mode='r+', - encoding="utf-8", dir=self.dl_dir, suffix=".json" + with tempfile.NamedTemporaryFile( + mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" ) as dcm2niix_config_file: self.export_dcm2niix_config_options(dcm2niix_config_file.name) workflow_params = { - "files": glob( - opj(tmp_dicom, "*", "*.dcm"), recursive=True - ), + "files": glob(opj(tmp_dicom, "*", "*.dcm"), recursive=True), "outdir": opj(self.dl_dir, str(self.shanoir_study_id)), "subjs": [subject_id], "converter": "dcm2niix", "heuristic": heuristic_file.name, "bids_options": "--bids", + # "with_prov": True, "dcmconfig": dcm2niix_config_file.name, "datalad": True, "minmeta": True, + "grouping": "all", # other options are too restrictive (tested on EMISEP) } if self.longitudinal: workflow_params["session"] = bids_seq_session workflow(**workflow_params) + # TODO add nipype logging into shanoir log file ? + # TODO use provenance option ? currently not working properly fp.close() def download(self): From 4aa85ce96d04dbd9786b90c6e83da5031210eb35 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Thu, 28 Mar 2024 15:42:47 +0100 Subject: [PATCH 06/86] + --- s2b_example_config.json | 4 +-- shanoir2bids_heudiconv.py | 56 ++++++++++++++++++++++----------------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 474faf7..264f23f 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -9,8 +9,8 @@ {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b3000_dir-AP_dwi"}, {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-AP_dwi"} ], - "dcm2niix":"/home/qduche/Software/dcm2niix_lnx/dcm2niix", - "dcm2niix_options": "-v 0 -z y", + "dcm2niix":"~/softs/miniconda3/bin/dcm2niix", + "dcm2niix_options": {"verbose": false,"compress": "y", "anon_bids": true}, "find_and_replace_subject": [ {"find":"VS_Aneravimm_", "replace": "VS"}, diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index dcb5b55..f0d224d 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -24,6 +24,7 @@ import shanoir_downloader from dotenv import load_dotenv from heudiconv.main import workflow + # import loggger used in heudiconv workflow from heudiconv.main import lgr @@ -172,7 +173,7 @@ def read_json_config_file(json_file): def generate_heuristic_file( - shanoir2bids_dict: object, path_heuristic_file: object + shanoir2bids_dict: object, path_heuristic_file: object, output_type ) -> None: """Generate heudiconv heuristic.py file from shanoir2bids mapping dict Parameters @@ -180,11 +181,18 @@ def generate_heuristic_file( shanoir2bids_dict : path_heuristic_file : path of the python heuristic file (.py) """ + if output_type == 'dicom': + outtype = '("dicom",)' + elif output_type == 'nifti': + outtype = '("nii.gz",)' + else: + outtype = '("dicom","nii.gz")' + heuristic = f"""from heudiconv.heuristics.reproin import create_key def create_bids_key(dataset): - template = create_key(subdir=dataset['bidsDir'],file_suffix=r"run-{{item:02d}}_" + dataset['bidsName'],outtype=("dicom","nii.gz")) + template = create_key(subdir=dataset['bidsDir'],file_suffix=r"run-{{item:02d}}_" + dataset['bidsName'],outtype={outtype}) return template def get_dataset_to_key_mapping(shanoir2bids): @@ -248,6 +256,7 @@ def __init__(self): self.shanoir_file_type = ( DEFAULT_SHANOIR_FILE_TYPE # Default download type (nifti/dicom) ) + self.output_file_type = DEFAULT_SHANOIR_FILE_TYPE self.json_config_file = None self.list_fars = [] # List of substrings to edit in subjects names self.dl_dir = None # download directory, where data will be stored @@ -298,6 +307,12 @@ def set_shanoir_file_type(self, shanoir_file_type): else: sys.exit("Unknown shanoir file type {}".format(shanoir_file_type)) + def set_output_file_type(self, output_file_type): + if output_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, 'both']: + self.shanoir_file_type = output_file_type + else: + sys.exit("Unknown shanoir file type {}".format(output_file_type)) + def set_shanoir_study_id(self, study_id): self.shanoir_study_id = study_id @@ -354,18 +369,6 @@ def set_download_directory(self, dl_dir): Path(self.dl_dir).mkdir(parents=True, exist_ok=True) self.set_log_filename() - def set_heuristic_file(self, path_heuristic_file): - if path_heuristic_file is None: - print(f"No heuristic file provided") - else: - filename, ext = ops(path_heuristic_file) - if ext != ".py": - print( - f"Provided heuristic file {path_heuristic_file} is not a .py file as expected" - ) - else: - self.heuristic_file = path_heuristic_file - def set_log_filename(self): curr_time = datetime.datetime.now() basename = "shanoir_downloader_{:04}{:02}{:02}_{:02}{:02}{:02}.log".format( @@ -582,7 +585,7 @@ def download_subject(self, subject_to_search): mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" ) as heuristic_file: # Generate Heudiconv heuristic file from configuration.json mapping - generate_heuristic_file(bids_mapping, heuristic_file.name) + generate_heuristic_file(bids_mapping, heuristic_file.name, output_type=self.output_file_type) with tempfile.NamedTemporaryFile( mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" ) as dcm2niix_config_file: @@ -641,13 +644,20 @@ def main(): default="shanoir.irisa.fr", help="The shanoir domain to query.", ) + # parser.add_argument( + # "-f", + # "--format", + # default="dicom", + # choices=["dicom"], + # help="The format to download.", + # ) parser.add_argument( - "-f", - "--format", - default="dicom", - choices=["dicom"], + "--outformat", + default="nifti", + choices=["nifti", "dicom", "both"], help="The format to download.", ) + shanoir_downloader.add_output_folder_argument(parser=parser, required=False) # Add the argument for the configuration file parser.add_argument( @@ -663,6 +673,7 @@ def main(): action="store_true", help="Toggle longitudinal approach.", ) + # parser.add_argument( # "-a", "--automri", action="store_true", help="Switch to automri file tree." # ) @@ -682,14 +693,11 @@ def main(): stb.set_json_config_file( json_file=args.config_file ) # path to json configuration file - stb.set_shanoir_file_type(shanoir_file_type=args.format) # Format (dicom or nifti) + stb.set_output_file_type(output_file_type=args.outformat) stb.set_download_directory( dl_dir=args.output_folder ) # output folder (if None a default directory is created) - # stb.set_heuristic_file(path_heuristic_file="/home/alpron/heuristic.py") - # stb.set_dcm2niix_config_files( - # path_dcm2niix_options_files="/home/alpron/dcm2niix_options.json" - # ) + if args.longitudinal: stb.toggle_longitudinal_version() # if args.automri: From 4656cc56da2964816581fb185aa89d7eef68b10c Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 8 Apr 2024 16:21:05 +0200 Subject: [PATCH 07/86] clean parser options --- shanoir2bids_heudiconv.py | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index f0d224d..1d58184 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -384,12 +384,6 @@ def set_log_filename(self): def toggle_longitudinal_version(self): self.longitudinal = True - def switch_to_automri_format(self): - self.to_automri_format = True - - def add_series_number_suffix(self): - self.add_sns = True - def configure_parser(self): """ Configure the parser and the configuration of the shanoir_downloader @@ -418,7 +412,6 @@ def download_subject(self, subject_to_search): # temporary directory containing dowloaded DICOM.zip files with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_dicom: with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_archive: - print(tmp_archive) # Loop on each sequence defined in the dictionary for seq in range(self.n_seq): # Isolate elements that are called many times @@ -644,13 +637,7 @@ def main(): default="shanoir.irisa.fr", help="The shanoir domain to query.", ) - # parser.add_argument( - # "-f", - # "--format", - # default="dicom", - # choices=["dicom"], - # help="The format to download.", - # ) + parser.add_argument( "--outformat", default="nifti", @@ -674,16 +661,6 @@ def main(): help="Toggle longitudinal approach.", ) - # parser.add_argument( - # "-a", "--automri", action="store_true", help="Switch to automri file tree." - # ) - # parser.add_argument( - # "-A", - # "--add_sns", - # action="store_true", - # help="Add series number suffix (compatible with -a)", - # ) - # Parse arguments args = parser.parse_args() # Start configuring the DownloadShanoirDatasetToBids class instance @@ -700,12 +677,6 @@ def main(): if args.longitudinal: stb.toggle_longitudinal_version() - # if args.automri: - # stb.switch_to_automri_format() - # if args.add_sns: - # if not args.automri: - # print("Warning : -A option is only compatible with -a option.") - # stb.add_series_number_suffix() stb.download() From d07faee1ca4707b9fb37fba5510e161e2c1f0e2d Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 9 Apr 2024 10:53:14 +0200 Subject: [PATCH 08/86] clean parser options --- s2b_example_config.json | 2 +- shanoir2bids_heudiconv.py | 35 ++++++++++++++--------------------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 264f23f..86e4041 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -9,7 +9,7 @@ {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b3000_dir-AP_dwi"}, {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-AP_dwi"} ], - "dcm2niix":"~/softs/miniconda3/bin/dcm2niix", + "dcm2niix":"/home/alpron/softs/miniconda3/bin/dcm2niix", "dcm2niix_options": {"verbose": false,"compress": "y", "anon_bids": true}, "find_and_replace_subject": [ diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index 1d58184..fe363ff 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -173,7 +173,7 @@ def read_json_config_file(json_file): def generate_heuristic_file( - shanoir2bids_dict: object, path_heuristic_file: object, output_type + shanoir2bids_dict: object, path_heuristic_file: object, output_type='("dicom","nii.gz")' ) -> None: """Generate heudiconv heuristic.py file from shanoir2bids mapping dict Parameters @@ -233,7 +233,6 @@ def infotodict(seqinfo): with open(path_heuristic_file, "w", encoding="utf-8") as file: file.write(heuristic) - file.close() pass @@ -253,10 +252,8 @@ def __init__(self): self.shanoir_username = None # Shanoir username self.shanoir_study_id = None # Shanoir study ID self.shanoir_session_id = None # Shanoir study ID - self.shanoir_file_type = ( - DEFAULT_SHANOIR_FILE_TYPE # Default download type (nifti/dicom) - ) - self.output_file_type = DEFAULT_SHANOIR_FILE_TYPE + self.shanoir_file_type = SHANOIR_FILE_TYPE_DICOM # Download File Type (DICOM) + self.output_file_type = DEFAULT_SHANOIR_FILE_TYPE # Default Export File Type (NIFTI) self.json_config_file = None self.list_fars = [] # List of substrings to edit in subjects names self.dl_dir = None # download directory, where data will be stored @@ -301,17 +298,11 @@ def set_json_config_file(self, json_file): self.set_date_from(date_from=date_from) self.set_date_to(date_to=date_to) - def set_shanoir_file_type(self, shanoir_file_type): - if shanoir_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI]: - self.shanoir_file_type = shanoir_file_type - else: - sys.exit("Unknown shanoir file type {}".format(shanoir_file_type)) - - def set_output_file_type(self, output_file_type): - if output_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, 'both']: - self.shanoir_file_type = output_file_type + def set_output_file_type(self, outfile_type): + if outfile_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, 'both']: + self.output_file_type = outfile_type else: - sys.exit("Unknown shanoir file type {}".format(output_file_type)) + sys.exit("Unknown output file type {}".format(outfile_type)) def set_shanoir_study_id(self, study_id): self.shanoir_study_id = study_id @@ -389,7 +380,11 @@ def configure_parser(self): Configure the parser and the configuration of the shanoir_downloader """ self.parser = shanoir_downloader.create_arg_parser() - shanoir_downloader.add_common_arguments(self.parser) + shanoir_downloader.add_username_argument(self.parser) + shanoir_downloader.add_domain_argument(self.parser) + self.parser.add_argument('-f', '--format', default='dicom', choices=['dicom'], + help='The format to download.') + shanoir_downloader.add_output_folder_argument(self.parser) shanoir_downloader.add_configuration_arguments(self.parser) shanoir_downloader.add_search_arguments(self.parser) shanoir_downloader.add_ids_arguments(self.parser) @@ -601,8 +596,6 @@ def download_subject(self, subject_to_search): workflow_params["session"] = bids_seq_session workflow(**workflow_params) - # TODO add nipype logging into shanoir log file ? - # TODO use provenance option ? currently not working properly fp.close() def download(self): @@ -640,7 +633,7 @@ def main(): parser.add_argument( "--outformat", - default="nifti", + default="both", choices=["nifti", "dicom", "both"], help="The format to download.", ) @@ -670,7 +663,7 @@ def main(): stb.set_json_config_file( json_file=args.config_file ) # path to json configuration file - stb.set_output_file_type(output_file_type=args.outformat) + stb.set_output_file_type(args.outformat) stb.set_download_directory( dl_dir=args.output_folder ) # output folder (if None a default directory is created) From 1ed0c6939462a5835a36f3d8ef5683c495d867ca Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 15 Apr 2024 14:11:01 +0200 Subject: [PATCH 09/86] added dcm2niix executable check --- shanoir2bids_heudiconv.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index fe363ff..fdd0820 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -261,6 +261,7 @@ def __init__(self): self.n_seq = 0 # Number of sequences in the shanoir2bids_dict self.log_fn = None self.dcm2niix_path = None # Path to the dcm2niix the user wants to use + self.actual_dcm2niix_path = shutil.which('dcm2niix') self.dcm2niix_opts = None # Options to add to the dcm2niix call self.date_from = None self.date_to = None @@ -375,6 +376,14 @@ def set_log_filename(self): def toggle_longitudinal_version(self): self.longitudinal = True + def is_correct_dcm2niix(self): + current_version = Path(self.actual_dcm2niix_path) + config_version = Path(self.dcm2niix_path) + if current_version is not None and config_version is not None: + return config_version.samefile(current_version) + else: + return False + def configure_parser(self): """ Configure the parser and the configuration of the shanoir_downloader @@ -670,8 +679,10 @@ def main(): if args.longitudinal: stb.toggle_longitudinal_version() - - stb.download() + if not stb.is_correct_dcm2niix(): + print(f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}") + else: + stb.download() if __name__ == "__main__": From 0139aed63de49affa270c23840033878699d99d7 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 22 Apr 2024 12:18:23 +0200 Subject: [PATCH 10/86] added back automri support option --- shanoir2bids_heudiconv.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index fdd0820..a8ec3e2 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -373,6 +373,9 @@ def set_log_filename(self): ) self.log_fn = opj(self.dl_dir, basename) + def switch_to_automri_format(self): + self.to_automri_format = True + def toggle_longitudinal_version(self): self.longitudinal = True @@ -603,6 +606,8 @@ def download_subject(self, subject_to_search): if self.longitudinal: workflow_params["session"] = bids_seq_session + if self.to_automri_format: + workflow_params["bids_options"] = None workflow(**workflow_params) fp.close() @@ -663,6 +668,8 @@ def main(): help="Toggle longitudinal approach.", ) + parser.add_argument('-a', '--automri', action='store_true', help='Switch to automri file tree.') + args = parser.parse_args() # Start configuring the DownloadShanoirDatasetToBids class instance @@ -679,6 +686,8 @@ def main(): if args.longitudinal: stb.toggle_longitudinal_version() + if args.automri: + stb.switch_to_automri_format() if not stb.is_correct_dcm2niix(): print(f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}") else: From 59a2f09b26454ac7039addb70fa6e343ad07aee3 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 22 Apr 2024 12:21:32 +0200 Subject: [PATCH 11/86] Revert "Merge branch 'bids-1.9.0-compliance' into heudiconv" This reverts commit 2c407217a0ea105f071f7f6c45a10458f28207b9, reversing changes made to 7f78df14532e45280b6c46323fd268c6fe9b1ca6. remove modifications made to configuration files to make them BIDS compliant --- s2b_example_config.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 86e4041..617ae20 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -3,11 +3,11 @@ "subjects": ["VS_Aneravimm_010", "VS_Aneravimm_011"], "data_to_bids": [ - {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", "bidsName": "acq-mprage_T1w"}, - {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "acq-hr_T2w"}, - {"datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", "bidsName": "task-restingState_acq-hipp_dir-AP_bold"}, - {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b3000_dir-AP_dwi"}, - {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-AP_dwi"} + {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", "bidsName": "t1w-mprage"}, + {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "t2-hr"}, + {"datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", "bidsName": "ap-hipp"}, + {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "cusp66-ap-b3000"}, + {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "cusp66-ap-b0"} ], "dcm2niix":"/home/alpron/softs/miniconda3/bin/dcm2niix", "dcm2niix_options": {"verbose": false,"compress": "y", "anon_bids": true}, From 0fca77ab6d01933488825e7090c4c65696dc5bf0 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 22 Apr 2024 12:46:40 +0200 Subject: [PATCH 12/86] added main dcm2niix configuration options into nipype format + documentation link as comment entry --- s2b_example_config.json | 61 ++++++++++++++++++++--------- s2b_example_config_EMISEP_long.json | 13 +++++- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 617ae20..0746b5e 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -1,19 +1,44 @@ { - "study_name": "Aneravimm", - "subjects": ["VS_Aneravimm_010", "VS_Aneravimm_011"], - "data_to_bids": - [ - {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", "bidsName": "t1w-mprage"}, - {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "t2-hr"}, - {"datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", "bidsName": "ap-hipp"}, - {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "cusp66-ap-b3000"}, - {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "cusp66-ap-b0"} - ], - "dcm2niix":"/home/alpron/softs/miniconda3/bin/dcm2niix", - "dcm2niix_options": {"verbose": false,"compress": "y", "anon_bids": true}, - "find_and_replace_subject": - [ - {"find":"VS_Aneravimm_", "replace": "VS"}, - {"find":"Vs_Aneravimm_", "replace": "VS"} - ] -} \ No newline at end of file + "study_name": "Aneravimm", + "subjects": ["VS_Aneravimm_010", "VS_Aneravimm_011"], + "data_to_bids": [ + { + "datasetName": "t1_mprage_sag_p2_iso", + "bidsDir": "anat", + "bidsName": "t1w-mprage", + }, + {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "t2-hr"}, + { + "datasetName": "Resting State_bold AP 1.6mm HIPP", + "bidsDir": "func", + "bidsName": "ap-hipp" + }, + { + "datasetName": "Diff cusp66 b3000 AP 1.5mm", + "bidsDir": "dwi", + "bidsName": "cusp66-ap-b3000" + }, + { + "datasetName": "Diff cusp66 b0 PA 1.5mm", + "bidsDir": "dwi", + "bidsName": "cusp66-ap-b0" + } + ], + "dcm2niix": "/home/alpron/softs/miniconda3/bin/dcm2niix", + "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", + "dcm2niix_options": { + "bids_format": true, + "anon_bids": true, + "compress": "y", + "compression": 5, + "crop": false, + "has_private": false, + "ignore_deriv": false, + "single_file": false, + "verbose": false + }, + "find_and_replace_subject": [ + {"find": "VS_Aneravimm_", "replace": "VS"}, + {"find": "Vs_Aneravimm_", "replace": "VS"} + ] +} diff --git a/s2b_example_config_EMISEP_long.json b/s2b_example_config_EMISEP_long.json index bc64ab7..c7bfb4a 100644 --- a/s2b_example_config_EMISEP_long.json +++ b/s2b_example_config_EMISEP_long.json @@ -64,7 +64,18 @@ ], "dcm2niix":"~/softs/miniconda3/bin/dcm2niix", - "dcm2niix_options": {"verbose": true,"compress": "y", "anon_bids": true} + "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", + "dcm2niix_options": { + "bids_format": true, + "anon_bids": true, + "compress": "y", + "compression": 5, + "crop": false, + "has_private": false, + "ignore_deriv": false, + "single_file": false, + "verbose": true + } } From a5b9d3144244d0c9a681778ec1f1502f168b38ed Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 22 Apr 2024 12:47:48 +0200 Subject: [PATCH 13/86] fix authors --- shanoir2bids_heudiconv.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index a8ec3e2..645b45e 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -4,8 +4,7 @@ The script is made to run for every project given some information provided by the user into a ".json" configuration file. More details regarding the configuration file in the Readme.md""" # Script to download and BIDS-like organize data on Shanoir using "shanoir_downloader.py" developed by Arthur Masson -# @Author: Malo Gaubert , Quentin Duché -# @Date: 24 Juin 2022 + import os from os.path import join as opj, splitext as ops, exists as ope, dirname as opd From 295bd5274da79eaa26bc6162da48ff07b00959c8 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 21 May 2024 10:39:00 +0200 Subject: [PATCH 14/86] [FIX]: typo --- s2b_example_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 0746b5e..e03f55d 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -5,7 +5,7 @@ { "datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", - "bidsName": "t1w-mprage", + "bidsName": "t1w-mprage" }, {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "t2-hr"}, { From 5064edb4cbdbf24d1fafad5dc711896154400fa2 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 21 May 2024 14:18:46 +0200 Subject: [PATCH 15/86] [ENH]: not working hack for heudiconv support --- shanoir2bids_heudiconv.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index 645b45e..8a20817 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -609,6 +609,19 @@ def download_subject(self, subject_to_search): workflow_params["bids_options"] = None workflow(**workflow_params) + if self.to_automri_format: + # horrible hack to adapt to automri ontology + dicoms = glob(opj(self.dl_dir, str(self.shanoir_study_id), "**", "*.dcm"), recursive=True) + niftis = glob(opj(self.dl_dir, str(self.shanoir_study_id), "**", "*.nii.gz"), recursive=True) + export_files = dicoms + niftis + to_modify_files = [f for f in export_files if not '.git' in f] + for f in to_modify_files: + new_file = f.replace('/' + subject_id + '/', '/' ) + new_file = new_file.replace('sub-','su_') + os.system('git mv ' + f + ' ' + new_file) + from datalad.api import save + save(path=opj(self.dl_dir, str(self.shanoir_study_id)), recursive=True, message='reformat into automri standart') + fp.close() def download(self): From 6ff27a564f278c353a10057483e65a7816199fe1 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 21 May 2024 14:22:54 +0200 Subject: [PATCH 16/86] [LINT]: black linting these files --- s2b_example_config.json | 16 +- s2b_example_config_EMISEP_long.json | 367 ++++++++++++++++++++++------ shanoir2bids_heudiconv.py | 67 +++-- 3 files changed, 354 insertions(+), 96 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index e03f55d..4e868fd 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -5,24 +5,24 @@ { "datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", - "bidsName": "t1w-mprage" + "bidsName": "t1w-mprage", }, {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "t2-hr"}, { "datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", - "bidsName": "ap-hipp" + "bidsName": "ap-hipp", }, { "datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", - "bidsName": "cusp66-ap-b3000" + "bidsName": "cusp66-ap-b3000", }, { "datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", - "bidsName": "cusp66-ap-b0" - } + "bidsName": "cusp66-ap-b0", + }, ], "dcm2niix": "/home/alpron/softs/miniconda3/bin/dcm2niix", "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", @@ -35,10 +35,10 @@ "has_private": false, "ignore_deriv": false, "single_file": false, - "verbose": false + "verbose": false, }, "find_and_replace_subject": [ {"find": "VS_Aneravimm_", "replace": "VS"}, - {"find": "Vs_Aneravimm_", "replace": "VS"} - ] + {"find": "Vs_Aneravimm_", "replace": "VS"}, + ], } diff --git a/s2b_example_config_EMISEP_long.json b/s2b_example_config_EMISEP_long.json index c7bfb4a..d000628 100644 --- a/s2b_example_config_EMISEP_long.json +++ b/s2b_example_config_EMISEP_long.json @@ -1,70 +1,299 @@ { - "study_name": "EMISEP", - "subjects": ["08-74"], - "session": "08-74 MO ENCEPHALE", - "data_to_bids": - [ - {"datasetName": "3D T1 MPRAGE", "bidsDir": "brain", "bidsName": "T1w", "bidsSession": "M00"}, - {"datasetName": "*3d flair*", "bidsDir": "brain", "bidsName": "FLAIR", "bidsSession": "M00"}, - {"datasetName": "*c1c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", "bidsSession": "M00"}, - {"datasetName": "*c1-c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", "bidsSession": "M00"}, - - {"datasetName": "*c4c7*", "bidsDir": "spinalCord", "bidsName": "t2ax_C4C7", "bidsSession": "M00"}, - {"datasetName": "*c4-c7*", "bidsDir": "spinalCord", "bidsName": "t2ax_C4C7", "bidsSession": "M00"}, - - - {"datasetName": "t2_tse_rr_p2_sag", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "t2_tse_rr_p2_sag2.5 te 81", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_384", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_p2", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "SAG T2 CERV", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "SAG T2 DORSAL", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - - {"datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "t2_tse_rr_p2_sag_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_384_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260_comp_ad", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_p2_comp_ad", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "SAG T2 TSE MOELLE", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - - - - {"datasetName": "t1_fl3d_sag_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_sag_p2_iso mt", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_tra_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_tra_p2_iso mt", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso mt tr38", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso mt tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso mt_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso tr38", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat_mt", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38_sans_sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_m0_iso tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_mt_iso tr38", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - - - {"datasetName": "3d_t1_mprage_sag", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso cerv", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso cerv_rr", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso ofsep", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_axial", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_cor", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_tra", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso ofsep_rr", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "T1 MPRAGE CERV", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"} - - ], - "dcm2niix":"~/softs/miniconda3/bin/dcm2niix", - "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", + "study_name": "EMISEP", + "subjects": ["08-74"], + "session": "08-74 MO ENCEPHALE", + "data_to_bids": [ + { + "datasetName": "3D T1 MPRAGE", + "bidsDir": "brain", + "bidsName": "T1w", + "bidsSession": "M00", + }, + { + "datasetName": "*3d flair*", + "bidsDir": "brain", + "bidsName": "FLAIR", + "bidsSession": "M00", + }, + { + "datasetName": "*c1c3*", + "bidsDir": "spinalCord", + "bidsName": "t2ax_C1C3", + "bidsSession": "M00", + }, + { + "datasetName": "*c1-c3*", + "bidsDir": "spinalCord", + "bidsName": "t2ax_C1C3", + "bidsSession": "M00", + }, + { + "datasetName": "*c4c7*", + "bidsDir": "spinalCord", + "bidsName": "t2ax_C4C7", + "bidsSession": "M00", + }, + { + "datasetName": "*c4-c7*", + "bidsDir": "spinalCord", + "bidsName": "t2ax_C4C7", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_rr_p2_sag", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_rr_p2_sag2.5 te 81", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_384", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_p2", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "SAG T2 CERV", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "SAG T2 DORSAL", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp_sp", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_rr_p2_sag_comp_sp", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_384_comp_sp", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260_comp_ad", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_p2_comp_ad", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "SAG T2 TSE MOELLE", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_sag_p2_iso", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_sag_p2_iso mt", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_tra_p2_iso", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_tra_p2_iso mt", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso mt tr38", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso mt tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso mt_tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso tr38", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat_mt", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38_sans_sat", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso_tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_m0_iso tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_mt_iso tr38", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "3d_t1_mprage_sag", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso cerv", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso cerv_rr", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso ofsep", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_axial", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_cor", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_tra", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso ofsep_rr", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "T1 MPRAGE CERV", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + ], + "dcm2niix": "~/softs/miniconda3/bin/dcm2niix", + "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", "dcm2niix_options": { "bids_format": true, "anon_bids": true, @@ -74,8 +303,6 @@ "has_private": false, "ignore_deriv": false, "single_file": false, - "verbose": true - } + "verbose": true, + }, } - - diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index 8a20817..19c7427 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -172,7 +172,9 @@ def read_json_config_file(json_file): def generate_heuristic_file( - shanoir2bids_dict: object, path_heuristic_file: object, output_type='("dicom","nii.gz")' + shanoir2bids_dict: object, + path_heuristic_file: object, + output_type='("dicom","nii.gz")', ) -> None: """Generate heudiconv heuristic.py file from shanoir2bids mapping dict Parameters @@ -180,9 +182,9 @@ def generate_heuristic_file( shanoir2bids_dict : path_heuristic_file : path of the python heuristic file (.py) """ - if output_type == 'dicom': + if output_type == "dicom": outtype = '("dicom",)' - elif output_type == 'nifti': + elif output_type == "nifti": outtype = '("nii.gz",)' else: outtype = '("dicom","nii.gz")' @@ -252,7 +254,9 @@ def __init__(self): self.shanoir_study_id = None # Shanoir study ID self.shanoir_session_id = None # Shanoir study ID self.shanoir_file_type = SHANOIR_FILE_TYPE_DICOM # Download File Type (DICOM) - self.output_file_type = DEFAULT_SHANOIR_FILE_TYPE # Default Export File Type (NIFTI) + self.output_file_type = ( + DEFAULT_SHANOIR_FILE_TYPE # Default Export File Type (NIFTI) + ) self.json_config_file = None self.list_fars = [] # List of substrings to edit in subjects names self.dl_dir = None # download directory, where data will be stored @@ -260,7 +264,7 @@ def __init__(self): self.n_seq = 0 # Number of sequences in the shanoir2bids_dict self.log_fn = None self.dcm2niix_path = None # Path to the dcm2niix the user wants to use - self.actual_dcm2niix_path = shutil.which('dcm2niix') + self.actual_dcm2niix_path = shutil.which("dcm2niix") self.dcm2niix_opts = None # Options to add to the dcm2niix call self.date_from = None self.date_to = None @@ -299,7 +303,7 @@ def set_json_config_file(self, json_file): self.set_date_to(date_to=date_to) def set_output_file_type(self, outfile_type): - if outfile_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, 'both']: + if outfile_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, "both"]: self.output_file_type = outfile_type else: sys.exit("Unknown output file type {}".format(outfile_type)) @@ -393,8 +397,13 @@ def configure_parser(self): self.parser = shanoir_downloader.create_arg_parser() shanoir_downloader.add_username_argument(self.parser) shanoir_downloader.add_domain_argument(self.parser) - self.parser.add_argument('-f', '--format', default='dicom', choices=['dicom'], - help='The format to download.') + self.parser.add_argument( + "-f", + "--format", + default="dicom", + choices=["dicom"], + help="The format to download.", + ) shanoir_downloader.add_output_folder_argument(self.parser) shanoir_downloader.add_configuration_arguments(self.parser) shanoir_downloader.add_search_arguments(self.parser) @@ -584,7 +593,9 @@ def download_subject(self, subject_to_search): mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" ) as heuristic_file: # Generate Heudiconv heuristic file from configuration.json mapping - generate_heuristic_file(bids_mapping, heuristic_file.name, output_type=self.output_file_type) + generate_heuristic_file( + bids_mapping, heuristic_file.name, output_type=self.output_file_type + ) with tempfile.NamedTemporaryFile( mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" ) as dcm2niix_config_file: @@ -611,16 +622,32 @@ def download_subject(self, subject_to_search): workflow(**workflow_params) if self.to_automri_format: # horrible hack to adapt to automri ontology - dicoms = glob(opj(self.dl_dir, str(self.shanoir_study_id), "**", "*.dcm"), recursive=True) - niftis = glob(opj(self.dl_dir, str(self.shanoir_study_id), "**", "*.nii.gz"), recursive=True) + dicoms = glob( + opj(self.dl_dir, str(self.shanoir_study_id), "**", "*.dcm"), + recursive=True, + ) + niftis = glob( + opj( + self.dl_dir, + str(self.shanoir_study_id), + "**", + "*.nii.gz", + ), + recursive=True, + ) export_files = dicoms + niftis - to_modify_files = [f for f in export_files if not '.git' in f] + to_modify_files = [f for f in export_files if not ".git" in f] for f in to_modify_files: - new_file = f.replace('/' + subject_id + '/', '/' ) - new_file = new_file.replace('sub-','su_') - os.system('git mv ' + f + ' ' + new_file) + new_file = f.replace("/" + subject_id + "/", "/") + new_file = new_file.replace("sub-", "su_") + os.system("git mv " + f + " " + new_file) from datalad.api import save - save(path=opj(self.dl_dir, str(self.shanoir_study_id)), recursive=True, message='reformat into automri standart') + + save( + path=opj(self.dl_dir, str(self.shanoir_study_id)), + recursive=True, + message="reformat into automri standart", + ) fp.close() @@ -680,7 +707,9 @@ def main(): help="Toggle longitudinal approach.", ) - parser.add_argument('-a', '--automri', action='store_true', help='Switch to automri file tree.') + parser.add_argument( + "-a", "--automri", action="store_true", help="Switch to automri file tree." + ) args = parser.parse_args() @@ -701,7 +730,9 @@ def main(): if args.automri: stb.switch_to_automri_format() if not stb.is_correct_dcm2niix(): - print(f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}") + print( + f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}" + ) else: stb.download() From ad7a6396f19ceb83ec2e075c39bcf53f81c0cfe5 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Wed, 22 May 2024 14:36:38 +0200 Subject: [PATCH 17/86] [FIX] modified heuristic file to support automri --- shanoir2bids_heudiconv.py | 121 +++++++++++++++++++++++++++----------- 1 file changed, 87 insertions(+), 34 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index 19c7427..6762a73 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -170,10 +170,88 @@ def read_json_config_file(json_file): date_to, ) +def generate_automri_heuristic_file( + shanoir2bids_dict, + path_heuristic_file, + subject, + output_type='("dicom","nii.gz")', + session=None): + if output_type == "dicom": + outtype = '("dicom",)' + elif output_type == "nifti": + outtype = '("nii.gz",)' + else: + outtype = '("dicom","nii.gz")' + + heuristic = f""" + def create_key( + subdir: Optional[str], + file_suffix: str, + outtype: tuple[str, ...] = ("nii.gz", "dicom"), + annotation_classes: None = None, + prefix: str = "", +) -> tuple[str, tuple[str, ...], None]: + if not subdir: + raise ValueError("subdir must be a valid format string") + template = os.path.join( + prefix, + "{subject}", + subdir, + "su_{subject}_%s" % file_suffix, + ) + return template, outtype, annotation_classes + + def create_bids_key(dataset): + + template = create_key(subdir=dataset['bidsDir'],file_suffix=r"run-{{item:02d}}_" + dataset['bidsName'],outtype={outtype}) + return template + + def get_dataset_to_key_mapping(shanoir2bids): + dataset_to_key = dict() + for dataset in shanoir2bids: + template = create_bids_key(dataset) + dataset_to_key[dataset['datasetName']] = template + return dataset_to_key + + def simplify_runs(info): + info_final = dict() + for key in info.keys(): + if len(info[key])==1: + new_template = key[0].replace('run-{{item:02d}}_','') + new_key = (new_template, key[1], key[2]) + info_final[new_key] = info[key] + else: + info_final[key] = info[key] + return info_final + + def infotodict(seqinfo): + + info = dict() + shanoir2bids = {shanoir2bids_dict} + + dataset_to_key = get_dataset_to_key_mapping(shanoir2bids) + for seq in seqinfo: + if seq.series_description in dataset_to_key.keys(): + key = dataset_to_key[seq.series_description] + if key in info.keys(): + info[key].append(seq.series_id) + else: + info[key] = [seq.series_id] + # remove run- key if not needed (one run only) + info_final = simplify_runs(info) + return info_final + """ + + with open(path_heuristic_file, "w", encoding="utf-8") as file: + file.write(heuristic) + pass + -def generate_heuristic_file( - shanoir2bids_dict: object, - path_heuristic_file: object, + +def generate_bids_heuristic_file( + shanoir2bids_dict, + + path_heuristic_file, output_type='("dicom","nii.gz")', ) -> None: """Generate heudiconv heuristic.py file from shanoir2bids mapping dict @@ -517,9 +595,11 @@ def download_subject(self, subject_to_search): # If the user has defined a list of edits to subject names... then do the find and replace for far in self.list_fars: su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) - # ID of the subject (sub-*) subject_id = su_id + + if self.to_automri_format: + subject_id = 'su_' + su_id # correct BIDS mapping of the searched dataset bids_seq_mapping = { "datasetName": item["datasetName"], @@ -593,7 +673,9 @@ def download_subject(self, subject_to_search): mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" ) as heuristic_file: # Generate Heudiconv heuristic file from configuration.json mapping - generate_heuristic_file( + if self.to_automri_format: + generate_automri_heuristic_file(bids_mapping,heuristic_file.name, subject_id,self.output_file_type, bids_seq_session) + generate_bids_heuristic_file( bids_mapping, heuristic_file.name, output_type=self.output_file_type ) with tempfile.NamedTemporaryFile( @@ -620,35 +702,6 @@ def download_subject(self, subject_to_search): workflow_params["bids_options"] = None workflow(**workflow_params) - if self.to_automri_format: - # horrible hack to adapt to automri ontology - dicoms = glob( - opj(self.dl_dir, str(self.shanoir_study_id), "**", "*.dcm"), - recursive=True, - ) - niftis = glob( - opj( - self.dl_dir, - str(self.shanoir_study_id), - "**", - "*.nii.gz", - ), - recursive=True, - ) - export_files = dicoms + niftis - to_modify_files = [f for f in export_files if not ".git" in f] - for f in to_modify_files: - new_file = f.replace("/" + subject_id + "/", "/") - new_file = new_file.replace("sub-", "su_") - os.system("git mv " + f + " " + new_file) - from datalad.api import save - - save( - path=opj(self.dl_dir, str(self.shanoir_study_id)), - recursive=True, - message="reformat into automri standart", - ) - fp.close() def download(self): From 7b87841f77b6ed37a008a1bb4161fffe821c2c2c Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Wed, 22 May 2024 14:58:17 +0200 Subject: [PATCH 18/86] [FIX] fixed typo introduced by black linting in json file --- s2b_example_config.json | 14 ++-- s2b_example_config_EMISEP_long.json | 102 ++++++++++++++-------------- 2 files changed, 58 insertions(+), 58 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 4e868fd..c8e3f13 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -5,23 +5,23 @@ { "datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", - "bidsName": "t1w-mprage", + "bidsName": "t1w-mprage" }, {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "t2-hr"}, { "datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", - "bidsName": "ap-hipp", + "bidsName": "ap-hipp" }, { "datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", - "bidsName": "cusp66-ap-b3000", + "bidsName": "cusp66-ap-b3000" }, { "datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", - "bidsName": "cusp66-ap-b0", + "bidsName": "cusp66-ap-b0" }, ], "dcm2niix": "/home/alpron/softs/miniconda3/bin/dcm2niix", @@ -35,10 +35,10 @@ "has_private": false, "ignore_deriv": false, "single_file": false, - "verbose": false, + "verbose": false }, "find_and_replace_subject": [ {"find": "VS_Aneravimm_", "replace": "VS"}, - {"find": "Vs_Aneravimm_", "replace": "VS"}, - ], + {"find": "Vs_Aneravimm_", "replace": "VS"} + ] } diff --git a/s2b_example_config_EMISEP_long.json b/s2b_example_config_EMISEP_long.json index d000628..e22a037 100644 --- a/s2b_example_config_EMISEP_long.json +++ b/s2b_example_config_EMISEP_long.json @@ -7,290 +7,290 @@ "datasetName": "3D T1 MPRAGE", "bidsDir": "brain", "bidsName": "T1w", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "*3d flair*", "bidsDir": "brain", "bidsName": "FLAIR", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "*c1c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "*c1-c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "*c4c7*", "bidsDir": "spinalCord", "bidsName": "t2ax_C4C7", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "*c4-c7*", "bidsDir": "spinalCord", "bidsName": "t2ax_C4C7", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_rr_p2_sag", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_rr_p2_sag2.5 te 81", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_384", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_p2", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "SAG T2 CERV", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "SAG T2 DORSAL", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_rr_p2_sag_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_384_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260_comp_ad", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_p2_comp_ad", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "SAG T2 TSE MOELLE", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_sag_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_sag_p2_iso mt", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_tra_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_tra_p2_iso mt", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso mt tr38", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso mt tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso mt_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso tr38", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat_mt", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38_sans_sat", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_m0_iso tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_mt_iso tr38", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "3d_t1_mprage_sag", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso cerv", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso cerv_rr", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso ofsep", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_axial", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_cor", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_tra", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso ofsep_rr", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "T1 MPRAGE CERV", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", - }, + "bidsSession": "M00" + } ], "dcm2niix": "~/softs/miniconda3/bin/dcm2niix", "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", @@ -303,6 +303,6 @@ "has_private": false, "ignore_deriv": false, "single_file": false, - "verbose": true, - }, + "verbose": true + } } From 8966d63990e6dae55b25f0d1a54bdb18b998910b Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Wed, 22 May 2024 16:13:06 +0200 Subject: [PATCH 19/86] [FIX] fixed typo introduced by black linting in json file --- s2b_example_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index c8e3f13..e03f55d 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -22,7 +22,7 @@ "datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "cusp66-ap-b0" - }, + } ], "dcm2niix": "/home/alpron/softs/miniconda3/bin/dcm2niix", "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", From 21cbdc2f0dfc5a8f3b2f6d242a3fa7a7454ef1fb Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Wed, 22 May 2024 16:14:11 +0200 Subject: [PATCH 20/86] [ENH]: v2 of automri generation. Not to be used (back up solution) use post processing parsing --- shanoir2bids_heudiconv.py | 129 +++++++++++++++++++++----------------- 1 file changed, 70 insertions(+), 59 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index 6762a73..c7c561c 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -170,12 +170,14 @@ def read_json_config_file(json_file): date_to, ) + def generate_automri_heuristic_file( - shanoir2bids_dict, - path_heuristic_file, - subject, - output_type='("dicom","nii.gz")', - session=None): + shanoir2bids_dict, + path_heuristic_file, + subject, + output_type='("dicom","nii.gz")', + session=None, +): if output_type == "dicom": outtype = '("dicom",)' elif output_type == "nifti": @@ -183,63 +185,63 @@ def generate_automri_heuristic_file( else: outtype = '("dicom","nii.gz")' - heuristic = f""" - def create_key( - subdir: Optional[str], - file_suffix: str, - outtype: tuple[str, ...] = ("nii.gz", "dicom"), - annotation_classes: None = None, - prefix: str = "", -) -> tuple[str, tuple[str, ...], None]: + heuristic = f"""import os + +def create_key( + subdir, + file_suffix, + outtype= ("nii.gz", "dicom"), + annotation_classes= None, + prefix= "", +): if not subdir: raise ValueError("subdir must be a valid format string") template = os.path.join( prefix, - "{subject}", subdir, - "su_{subject}_%s" % file_suffix, + "{subject}_%s" % file_suffix, ) return template, outtype, annotation_classes - def create_bids_key(dataset): - - template = create_key(subdir=dataset['bidsDir'],file_suffix=r"run-{{item:02d}}_" + dataset['bidsName'],outtype={outtype}) - return template - - def get_dataset_to_key_mapping(shanoir2bids): - dataset_to_key = dict() - for dataset in shanoir2bids: - template = create_bids_key(dataset) - dataset_to_key[dataset['datasetName']] = template - return dataset_to_key - - def simplify_runs(info): - info_final = dict() - for key in info.keys(): - if len(info[key])==1: - new_template = key[0].replace('run-{{item:02d}}_','') - new_key = (new_template, key[1], key[2]) - info_final[new_key] = info[key] - else: - info_final[key] = info[key] - return info_final +def create_bids_key(dataset): - def infotodict(seqinfo): + template = create_key(subdir=dataset['bidsDir'],file_suffix=r"run-{{item:02d}}_" + dataset['bidsName'],outtype={outtype}) + return template - info = dict() - shanoir2bids = {shanoir2bids_dict} +def get_dataset_to_key_mapping(shanoir2bids): + dataset_to_key = dict() + for dataset in shanoir2bids: + template = create_bids_key(dataset) + dataset_to_key[dataset['datasetName']] = template + return dataset_to_key - dataset_to_key = get_dataset_to_key_mapping(shanoir2bids) - for seq in seqinfo: - if seq.series_description in dataset_to_key.keys(): - key = dataset_to_key[seq.series_description] - if key in info.keys(): - info[key].append(seq.series_id) - else: - info[key] = [seq.series_id] - # remove run- key if not needed (one run only) - info_final = simplify_runs(info) - return info_final +def simplify_runs(info): + info_final = dict() + for key in info.keys(): + if len(info[key])==1: + new_template = key[0].replace('run-{{item:02d}}_','') + new_key = (new_template, key[1], key[2]) + info_final[new_key] = info[key] + else: + info_final[key] = info[key] + return info_final + +def infotodict(seqinfo): + + info = dict() + shanoir2bids = {shanoir2bids_dict} + + dataset_to_key = get_dataset_to_key_mapping(shanoir2bids) + for seq in seqinfo: + if seq.series_description in dataset_to_key.keys(): + key = dataset_to_key[seq.series_description] + if key in info.keys(): + info[key].append(seq.series_id) + else: + info[key] = [seq.series_id] + # remove run- key if not needed (one run only) + info_final = simplify_runs(info) + return info_final """ with open(path_heuristic_file, "w", encoding="utf-8") as file: @@ -247,10 +249,8 @@ def infotodict(seqinfo): pass - def generate_bids_heuristic_file( shanoir2bids_dict, - path_heuristic_file, output_type='("dicom","nii.gz")', ) -> None: @@ -599,7 +599,7 @@ def download_subject(self, subject_to_search): subject_id = su_id if self.to_automri_format: - subject_id = 'su_' + su_id + subject_id = "su_" + su_id # correct BIDS mapping of the searched dataset bids_seq_mapping = { "datasetName": item["datasetName"], @@ -613,7 +613,9 @@ def download_subject(self, subject_to_search): "bids_session_id" ] = bids_seq_session else: - bids_seq_mapping["bids_session_id"] = None + bids_seq_session = None + + bids_seq_mapping["bids_session_id"] = bids_seq_session bids_mapping.append(bids_seq_mapping) @@ -674,10 +676,17 @@ def download_subject(self, subject_to_search): ) as heuristic_file: # Generate Heudiconv heuristic file from configuration.json mapping if self.to_automri_format: - generate_automri_heuristic_file(bids_mapping,heuristic_file.name, subject_id,self.output_file_type, bids_seq_session) - generate_bids_heuristic_file( - bids_mapping, heuristic_file.name, output_type=self.output_file_type - ) + generate_automri_heuristic_file( + bids_mapping, + heuristic_file.name, + subject_id, + self.output_file_type, + bids_seq_session, + ) + else: + generate_bids_heuristic_file( + bids_mapping, heuristic_file.name, output_type=self.output_file_type + ) with tempfile.NamedTemporaryFile( mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" ) as dcm2niix_config_file: @@ -698,7 +707,9 @@ def download_subject(self, subject_to_search): if self.longitudinal: workflow_params["session"] = bids_seq_session + print('toto') if self.to_automri_format: + print('bubu') workflow_params["bids_options"] = None workflow(**workflow_params) From 3c46df24e9f986a7393ed5b4e4d5d33ce7b30c69 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 28 May 2024 15:04:07 +0200 Subject: [PATCH 21/86] [FIX]: removed automri support and linting --- shanoir2bids_heudiconv.py | 108 ++------------------------------------ 1 file changed, 3 insertions(+), 105 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index c7c561c..f7a8c48 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -171,84 +171,6 @@ def read_json_config_file(json_file): ) -def generate_automri_heuristic_file( - shanoir2bids_dict, - path_heuristic_file, - subject, - output_type='("dicom","nii.gz")', - session=None, -): - if output_type == "dicom": - outtype = '("dicom",)' - elif output_type == "nifti": - outtype = '("nii.gz",)' - else: - outtype = '("dicom","nii.gz")' - - heuristic = f"""import os - -def create_key( - subdir, - file_suffix, - outtype= ("nii.gz", "dicom"), - annotation_classes= None, - prefix= "", -): - if not subdir: - raise ValueError("subdir must be a valid format string") - template = os.path.join( - prefix, - subdir, - "{subject}_%s" % file_suffix, - ) - return template, outtype, annotation_classes - -def create_bids_key(dataset): - - template = create_key(subdir=dataset['bidsDir'],file_suffix=r"run-{{item:02d}}_" + dataset['bidsName'],outtype={outtype}) - return template - -def get_dataset_to_key_mapping(shanoir2bids): - dataset_to_key = dict() - for dataset in shanoir2bids: - template = create_bids_key(dataset) - dataset_to_key[dataset['datasetName']] = template - return dataset_to_key - -def simplify_runs(info): - info_final = dict() - for key in info.keys(): - if len(info[key])==1: - new_template = key[0].replace('run-{{item:02d}}_','') - new_key = (new_template, key[1], key[2]) - info_final[new_key] = info[key] - else: - info_final[key] = info[key] - return info_final - -def infotodict(seqinfo): - - info = dict() - shanoir2bids = {shanoir2bids_dict} - - dataset_to_key = get_dataset_to_key_mapping(shanoir2bids) - for seq in seqinfo: - if seq.series_description in dataset_to_key.keys(): - key = dataset_to_key[seq.series_description] - if key in info.keys(): - info[key].append(seq.series_id) - else: - info[key] = [seq.series_id] - # remove run- key if not needed (one run only) - info_final = simplify_runs(info) - return info_final - """ - - with open(path_heuristic_file, "w", encoding="utf-8") as file: - file.write(heuristic) - pass - - def generate_bids_heuristic_file( shanoir2bids_dict, path_heuristic_file, @@ -454,9 +376,6 @@ def set_log_filename(self): ) self.log_fn = opj(self.dl_dir, basename) - def switch_to_automri_format(self): - self.to_automri_format = True - def toggle_longitudinal_version(self): self.longitudinal = True @@ -598,8 +517,6 @@ def download_subject(self, subject_to_search): # ID of the subject (sub-*) subject_id = su_id - if self.to_automri_format: - subject_id = "su_" + su_id # correct BIDS mapping of the searched dataset bids_seq_mapping = { "datasetName": item["datasetName"], @@ -675,18 +592,9 @@ def download_subject(self, subject_to_search): mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" ) as heuristic_file: # Generate Heudiconv heuristic file from configuration.json mapping - if self.to_automri_format: - generate_automri_heuristic_file( - bids_mapping, - heuristic_file.name, - subject_id, - self.output_file_type, - bids_seq_session, - ) - else: - generate_bids_heuristic_file( - bids_mapping, heuristic_file.name, output_type=self.output_file_type - ) + generate_bids_heuristic_file( + bids_mapping, heuristic_file.name, output_type=self.output_file_type + ) with tempfile.NamedTemporaryFile( mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" ) as dcm2niix_config_file: @@ -707,10 +615,6 @@ def download_subject(self, subject_to_search): if self.longitudinal: workflow_params["session"] = bids_seq_session - print('toto') - if self.to_automri_format: - print('bubu') - workflow_params["bids_options"] = None workflow(**workflow_params) fp.close() @@ -771,10 +675,6 @@ def main(): help="Toggle longitudinal approach.", ) - parser.add_argument( - "-a", "--automri", action="store_true", help="Switch to automri file tree." - ) - args = parser.parse_args() # Start configuring the DownloadShanoirDatasetToBids class instance @@ -791,8 +691,6 @@ def main(): if args.longitudinal: stb.toggle_longitudinal_version() - if args.automri: - stb.switch_to_automri_format() if not stb.is_correct_dcm2niix(): print( f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}" From c3c03cfc354421de3efb94b64142e3a44cd959a5 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 3 Jun 2024 11:34:30 +0200 Subject: [PATCH 22/86] [FIX]: removed temporary directories (not supported by python <3.11) by manual creation and cleaning or not if debug --- shanoir2bids_heudiconv.py | 388 ++++++++++++++++++++------------------ 1 file changed, 202 insertions(+), 186 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index f7a8c48..35ca063 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -94,6 +94,14 @@ def banner_msg(msg): If you want to do so, add key "{key}" to you Shanoir2BIDS configuration file :""" +def create_tmp_directory(path_temporary_directory): + tmp_dir = Path(path_temporary_directory) + if tmp_dir.exists(): + shutil.rmtree(tmp_dir) + tmp_dir.mkdir(parents=True) + pass + + def check_date_format(date_to_format): # TRUE FORMAT should be: date_format = 'Y-m-dTH:M:SZ' try: @@ -270,9 +278,10 @@ def __init__(self): self.date_to = None self.longitudinal = False self.to_automri_format = ( - False # Special filenames for automri (close to BIDS format) + False # Special filenames for automri (No longer used ! --> BIDS format) ) self.add_sns = False # Add series number suffix to filename + self.debug_mode = False # No debug mode by default def set_json_config_file(self, json_file): """ @@ -421,203 +430,200 @@ def download_subject(self, subject_to_search): # Real Shanoir2Bids mapping (handle case when solr search term are included) bids_mapping = [] - # temporary directory containing dowloaded DICOM.zip files - with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_dicom: - with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_archive: - # Loop on each sequence defined in the dictionary - for seq in range(self.n_seq): - # Isolate elements that are called many times - shanoir_seq_name = self.shanoir2bids_dict[seq][ - K_DS_NAME - ] # Shanoir sequence name (OLD) - bids_seq_subdir = self.shanoir2bids_dict[seq][ - K_BIDS_DIR - ] # Sequence BIDS subdirectory name (NEW) - bids_seq_name = self.shanoir2bids_dict[seq][ - K_BIDS_NAME - ] # Sequence BIDS nickname (NEW) - if self.longitudinal: - bids_seq_session = self.shanoir2bids_dict[seq][ - K_BIDS_SES - ] # Sequence BIDS nickname (NEW) - - # Print message concerning the sequence that is being downloaded - print( - "\t-", - bids_seq_name, - subject_to_search, - "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", - ) - # Initialize the parser - search_txt = ( - "studyName:" - + self.shanoir_study_id.replace(" ", "?") - + " AND datasetName:" - + shanoir_seq_name.replace(" ", "?") - + " AND subjectName:" - + subject_to_search.replace(" ", "?") - + " AND examinationComment:" - + self.shanoir_session_id.replace(" ", "*") - + " AND examinationDate:[" - + self.date_from - + " TO " - + self.date_to - + "]" + # Manual temporary directories containing dowloaded DICOM.zip and extracted files + # (temporary directories that can be kept are not supported by pythn <3.1 + tmp_dicom = Path(self.dl_dir).joinpath("tmp_dicoms", subject_to_search) + tmp_archive = Path(self.dl_dir).joinpath( + "tmp_archived_dicoms", subject_to_search + ) + create_tmp_directory(tmp_archive) + create_tmp_directory(tmp_dicom) + + # Loop on each sequence defined in the dictionary + for seq in range(self.n_seq): + # Isolate elements that are called many times + shanoir_seq_name = self.shanoir2bids_dict[seq][ + K_DS_NAME + ] # Shanoir sequence name (OLD) + bids_seq_subdir = self.shanoir2bids_dict[seq][ + K_BIDS_DIR + ] # Sequence BIDS subdirectory name (NEW) + bids_seq_name = self.shanoir2bids_dict[seq][ + K_BIDS_NAME + ] # Sequence BIDS nickname (NEW) + if self.longitudinal: + bids_seq_session = self.shanoir2bids_dict[seq][ + K_BIDS_SES + ] # Sequence BIDS nickname (NEW) + + # Print message concerning the sequence that is being downloaded + print( + "\t-", + bids_seq_name, + subject_to_search, + "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", + ) + + # Initialize the parser + search_txt = ( + "studyName:" + + self.shanoir_study_id.replace(" ", "?") + + " AND datasetName:" + + shanoir_seq_name.replace(" ", "?") + + " AND subjectName:" + + subject_to_search.replace(" ", "?") + + " AND examinationComment:" + + self.shanoir_session_id.replace(" ", "*") + + " AND examinationDate:[" + + self.date_from + + " TO " + + self.date_to + + "]" + ) + + args = self.parser.parse_args( + [ + "-u", + self.shanoir_username, + "-d", + self.shanoir_domaine, + "-of", + tmp_archive, + "-em", + "-st", + search_txt, + "-s", + "200", + "-f", + self.shanoir_file_type, + "-so", + "id,ASC", + "-t", + "500", + ] + ) # Increase time out for heavy files + + config = shanoir_downloader.initialize(args) + response = shanoir_downloader.solr_search(config, args) + + # From response, process the data + # Print the number of items found and a list of these items + if response.status_code == 200: + # Invoke shanoir_downloader to download all the data + shanoir_downloader.download_search_results(config, args, response) + + if len(response.json()["content"]) == 0: + warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns +a result on the website. +Search Text : "{}" \n""".format( + search_txt ) + print(warn_msg) + fp.write(warn_msg) + else: + for item in response.json()["content"]: + # Define subject_id + su_id = item["subjectName"] + # If the user has defined a list of edits to subject names... then do the find and replace + for far in self.list_fars: + su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) + # ID of the subject (sub-*) + subject_id = su_id + + # correct BIDS mapping of the searched dataset + bids_seq_mapping = { + "datasetName": item["datasetName"], + "bidsDir": bids_seq_subdir, + "bidsName": bids_seq_name, + "bids_subject_id": subject_id, + } + + if self.longitudinal: + bids_seq_mapping["bids_session_id"] = bids_seq_session + else: + bids_seq_session = None - args = self.parser.parse_args( - [ - "-u", - self.shanoir_username, - "-d", - self.shanoir_domaine, - "-of", - tmp_archive, - "-em", - "-st", - search_txt, - "-s", - "200", - "-f", - self.shanoir_file_type, - "-so", - "id,ASC", - "-t", - "500", - ] - ) # Increase time out for heavy files + bids_seq_mapping["bids_session_id"] = bids_seq_session - config = shanoir_downloader.initialize(args) - response = shanoir_downloader.solr_search(config, args) + bids_mapping.append(bids_seq_mapping) - # From response, process the data - # Print the number of items found and a list of these items - if response.status_code == 200: - # Invoke shanoir_downloader to download all the data - shanoir_downloader.download_search_results( - config, args, response + # Write the information on the data in the log file + fp.write("- datasetId = " + str(item["datasetId"]) + "\n") + fp.write(" -- studyName: " + item["studyName"] + "\n") + fp.write(" -- subjectName: " + item["subjectName"] + "\n") + fp.write(" -- session: " + item["examinationComment"] + "\n") + fp.write(" -- datasetName: " + item["datasetName"] + "\n") + fp.write( + " -- examinationDate: " + item["examinationDate"] + "\n" ) + fp.write(" >> Downloading archive OK\n") + + # Extract the downloaded archive + dl_archive = glob(opj(tmp_archive, "*" + item["id"] + "*.zip"))[ + 0 + ] + with zipfile.ZipFile(dl_archive, "r") as zip_ref: + extraction_dir = opj(tmp_dicom, item["id"]) + zip_ref.extractall(extraction_dir) - if len(response.json()["content"]) == 0: - warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns - a result on the website. - Search Text : "{}" \n""".format( - search_txt - ) - print(warn_msg) - fp.write(warn_msg) - else: - for item in response.json()["content"]: - # Define subject_id - su_id = item["subjectName"] - # If the user has defined a list of edits to subject names... then do the find and replace - for far in self.list_fars: - su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) - # ID of the subject (sub-*) - subject_id = su_id - - # correct BIDS mapping of the searched dataset - bids_seq_mapping = { - "datasetName": item["datasetName"], - "bidsDir": bids_seq_subdir, - "bidsName": bids_seq_name, - "bids_subject_id": subject_id, - } - - if self.longitudinal: - bids_seq_mapping[ - "bids_session_id" - ] = bids_seq_session - else: - bids_seq_session = None - - bids_seq_mapping["bids_session_id"] = bids_seq_session - - bids_mapping.append(bids_seq_mapping) - - # Write the information on the data in the log file - fp.write( - "- datasetId = " + str(item["datasetId"]) + "\n" - ) - fp.write(" -- studyName: " + item["studyName"] + "\n") - fp.write( - " -- subjectName: " + item["subjectName"] + "\n" - ) - fp.write( - " -- session: " + item["examinationComment"] + "\n" - ) - fp.write( - " -- datasetName: " + item["datasetName"] + "\n" - ) - fp.write( - " -- examinationDate: " - + item["examinationDate"] - + "\n" - ) - fp.write(" >> Downloading archive OK\n") - - # Extract the downloaded archive - dl_archive = glob( - opj(tmp_archive, "*" + item["id"] + "*.zip") - )[0] - with zipfile.ZipFile(dl_archive, "r") as zip_ref: - extraction_dir = opj(tmp_dicom, item["id"]) - zip_ref.extractall(extraction_dir) - - fp.write( - " >> Extraction of all files from archive '" - + dl_archive - + " into " - + extraction_dir - + "\n" - ) - - elif response.status_code == 204: - banner_msg("ERROR : No file found!") - fp.write(" >> ERROR : No file found!\n") - else: - banner_msg( - "ERROR : Returned by the request: status of the response = " - + response.status_code - ) fp.write( - " >> ERROR : Returned by the request: status of the response = " - + str(response.status_code) + " >> Extraction of all files from archive '" + + dl_archive + + " into " + + extraction_dir + "\n" ) - # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options - with tempfile.NamedTemporaryFile( - mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" - ) as heuristic_file: - # Generate Heudiconv heuristic file from configuration.json mapping - generate_bids_heuristic_file( - bids_mapping, heuristic_file.name, output_type=self.output_file_type + elif response.status_code == 204: + banner_msg("ERROR : No file found!") + fp.write(" >> ERROR : No file found!\n") + else: + banner_msg( + "ERROR : Returned by the request: status of the response = " + + response.status_code ) - with tempfile.NamedTemporaryFile( - mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" - ) as dcm2niix_config_file: - self.export_dcm2niix_config_options(dcm2niix_config_file.name) - workflow_params = { - "files": glob(opj(tmp_dicom, "*", "*.dcm"), recursive=True), - "outdir": opj(self.dl_dir, str(self.shanoir_study_id)), - "subjs": [subject_id], - "converter": "dcm2niix", - "heuristic": heuristic_file.name, - "bids_options": "--bids", - # "with_prov": True, - "dcmconfig": dcm2niix_config_file.name, - "datalad": True, - "minmeta": True, - "grouping": "all", # other options are too restrictive (tested on EMISEP) - } - - if self.longitudinal: - workflow_params["session"] = bids_seq_session - - workflow(**workflow_params) - fp.close() + fp.write( + " >> ERROR : Returned by the request: status of the response = " + + str(response.status_code) + + "\n" + ) + + # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options + with tempfile.NamedTemporaryFile( + mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" + ) as heuristic_file: + # Generate Heudiconv heuristic file from configuration.json mapping + generate_bids_heuristic_file( + bids_mapping, heuristic_file.name, output_type=self.output_file_type + ) + with tempfile.NamedTemporaryFile( + mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" + ) as dcm2niix_config_file: + self.export_dcm2niix_config_options(dcm2niix_config_file.name) + workflow_params = { + "files": glob(opj(tmp_dicom, "*", "*.dcm"), recursive=True), + "outdir": opj(self.dl_dir, str(self.shanoir_study_id)), + "subjs": [subject_id], + "converter": "dcm2niix", + "heuristic": heuristic_file.name, + "bids_options": "--bids", + # "with_prov": True, + "dcmconfig": dcm2niix_config_file.name, + "datalad": True, + "minmeta": True, + "grouping": "all", # other options are too restrictive (tested on EMISEP) + } + + if self.longitudinal: + workflow_params["session"] = bids_seq_session + + workflow(**workflow_params) + fp.close() + if not self.debug_mode: + shutil.rmtree(tmp_archive) + shutil.rmtree(tmp_dicom) + def download(self): """ @@ -674,6 +680,12 @@ def main(): action="store_true", help="Toggle longitudinal approach.", ) + parser.add_argument( + "--debug", + required=False, + action="store_true", + help="Toggle debug mode (keep temporary directories)", + ) args = parser.parse_args() @@ -689,8 +701,12 @@ def main(): dl_dir=args.output_folder ) # output folder (if None a default directory is created) + if args.debug: + stb.debug_mode = True + if args.longitudinal: stb.toggle_longitudinal_version() + if not stb.is_correct_dcm2niix(): print( f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}" From 005c9b704c24d31c49fb1d06bc3bdb820f227bce Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 3 Jun 2024 13:29:11 +0200 Subject: [PATCH 23/86] [FIX]: --- shanoir2bids_heudiconv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index 35ca063..cccb35f 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -489,7 +489,7 @@ def download_subject(self, subject_to_search): "-d", self.shanoir_domaine, "-of", - tmp_archive, + str(tmp_archive), "-em", "-st", search_txt, @@ -621,8 +621,8 @@ def download_subject(self, subject_to_search): workflow(**workflow_params) fp.close() if not self.debug_mode: - shutil.rmtree(tmp_archive) - shutil.rmtree(tmp_dicom) + shutil.rmtree(tmp_archive.parent) + shutil.rmtree(tmp_dicom.parent) def download(self): From b3dfb287273f1f16631d4a8729e2ba9689bbe900 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Fri, 22 Mar 2024 16:58:05 +0100 Subject: [PATCH 24/86] [ENH]: DICOM to NIFTI BIDS conversion with heuristic generated from configuration.json file. Configuration file was slightly modified to handle dcm2niix options following nipype DC2NIIX attributes --- s2b_example_config_EMISEP_long.json | 4 +- shanoir2bidsv1.py | 672 ++++++++++++++++++++++++++++ 2 files changed, 674 insertions(+), 2 deletions(-) create mode 100755 shanoir2bidsv1.py diff --git a/s2b_example_config_EMISEP_long.json b/s2b_example_config_EMISEP_long.json index c024c5e..cf6f037 100644 --- a/s2b_example_config_EMISEP_long.json +++ b/s2b_example_config_EMISEP_long.json @@ -63,8 +63,8 @@ {"datasetName": "T1 MPRAGE CERV", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"} ], - "dcm2niix":"/home/mgaubert/Software/dcm2niix/dcm2niix_v1.0.20211006/dcm2niix", - "dcm2niix_options": "-v 0 -z y -ba n" + "dcm2niix":"~/softs/miniconda3/bin/dcm2niix", + "dcm2niix_options": {"verbose": true,"compress": "y", "anon_bids": false} } diff --git a/shanoir2bidsv1.py b/shanoir2bidsv1.py new file mode 100755 index 0000000..7016425 --- /dev/null +++ b/shanoir2bidsv1.py @@ -0,0 +1,672 @@ +#!/usr/bin/env python3 +DESCRIPTION = """ +shanoir2bids.py is a script that allows to download a Shanoir dataset and organise it as a BIDS data structure. + The script is made to run for every project given some information provided by the user into a ".json" + configuration file. More details regarding the configuration file in the Readme.md""" +# Script to download and BIDS-like organize data on Shanoir using "shanoir_downloader.py" developed by Arthur Masson +# @Author: Malo Gaubert , Quentin Duché +# @Date: 24 Juin 2022 +import os +import sys +import zipfile +import json +import shutil +import shanoir_downloader +from dotenv import load_dotenv +from pathlib import Path +from os.path import join as opj, splitext as ops, exists as ope, dirname as opd +from glob import glob +from time import time +import datetime +from dateutil import parser + +from heudiconv.main import workflow + + +# Load environment variables +load_dotenv(dotenv_path=opj(opd(__file__), ".env")) + + +def banner_msg(msg): + """ + Print a message framed by a banner of "*" characters + :param msg: + """ + banner = "*" * (len(msg) + 6) + print(banner + "\n* ", msg, " *\n" + banner) + + +# Keys for json configuration file +K_JSON_STUDY_NAME = "study_name" +K_JSON_L_SUBJECTS = "subjects" +K_JSON_SESSION = "session" +K_JSON_DATA_DICT = "data_to_bids" +K_JSON_FIND_AND_REPLACE = "find_and_replace_subject" +K_DCM2NIIX_PATH = "dcm2niix" +K_DCM2NIIX_OPTS = "dcm2niix_options" +K_FIND = "find" +K_REPLACE = "replace" +K_JSON_DATE_FROM = ( + "date_from" # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] +) +K_JSON_DATE_TO = ( + "date_to" # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] +) +LIST_MANDATORY_KEYS_JSON = [K_JSON_STUDY_NAME, K_JSON_L_SUBJECTS, K_JSON_DATA_DICT] +LIST_AUTHORIZED_KEYS_JSON = LIST_MANDATORY_KEYS_JSON + [ + K_DCM2NIIX_PATH, + K_DCM2NIIX_OPTS, + K_JSON_DATE_FROM, + K_JSON_DATE_TO, + K_JSON_SESSION, +] + +# Define keys for data dictionary +K_BIDS_NAME = "bidsName" +K_BIDS_DIR = "bidsDir" +K_BIDS_SES = "bidsSession" +K_DS_NAME = "datasetName" + +# Define Extensions that are dealt so far by (#todo : think of other possible extensions ?) +NIFTI = ".nii" +NIIGZ = ".nii.gz" +JSON = ".json" +BVAL = ".bval" +BVEC = ".bvec" +DCM = ".dcm" + +# Shanoir parameters +SHANOIR_FILE_TYPE_NIFTI = "nifti" +SHANOIR_FILE_TYPE_DICOM = "dicom" +DEFAULT_SHANOIR_FILE_TYPE = SHANOIR_FILE_TYPE_NIFTI + +# Define error and warning messages when call to dcm2niix is not well configured in the json file +DCM2NIIX_ERR_MSG = """ERROR !! +Conversion from DICOM to nifti can not be performed. +Please provide path to your favorite dcm2niix version in your Shanoir2BIDS .json configuration file. +Add key "{key}" with the absolute path to dcm2niix version to the following file : """ +DCM2NIIX_WARN_MSG = """WARNING. You did not provide any option to the dcm2niix call. +If you want to do so, add key "{key}" to you Shanoir2BIDS configuration file :""" + + +def check_date_format(date_to_format): + # TRUE FORMAT should be: date_format = 'Y-m-dTH:M:SZ' + try: + parser.parse(date_to_format) + # If the date validation goes wrong + except ValueError: + print( + "Incorrect data format, should be YYYY-MM-DDTHH:MM:SSZ (for example: 2020-02-19T00:00:00Z)" + ) + + +def read_json_config_file(json_file): + """ + Reads a json configuration file and checks whether mandatory keys for specifying the transformation from a + Shanoir dataset to a BIDS dataset is present. + :param json_file: str, path to a json configuration file + :return: + """ + f = open(json_file) + data = json.load(f) + # Check keys + for key in data.keys(): + if not key in LIST_AUTHORIZED_KEYS_JSON: + print('Unknown key "{}" for data dictionary'.format(key)) + for key in LIST_MANDATORY_KEYS_JSON: + if not key in data.keys(): + sys.exit('Error, missing key "{}" in data dictionary'.format(key)) + + # Sets the mandatory fields for the instance of the class + study_id = data[K_JSON_STUDY_NAME] + subjects = data[K_JSON_L_SUBJECTS] + data_dict = data[K_JSON_DATA_DICT] + + # Default non-mandatory options + list_fars = [] + dcm2niix_path = None + dcm2niix_opts = None + date_from = "*" + date_to = "*" + session_id = "*" + + if K_JSON_FIND_AND_REPLACE in data.keys(): + list_fars = data[K_JSON_FIND_AND_REPLACE] + if K_DCM2NIIX_PATH in data.keys(): + dcm2niix_path = data[K_DCM2NIIX_PATH] + if K_DCM2NIIX_OPTS in data.keys(): + dcm2niix_opts = data[K_DCM2NIIX_OPTS] + if K_JSON_DATE_FROM in data.keys(): + if data[K_JSON_DATE_FROM] == "": + data_from = "*" + else: + date_from = data[K_JSON_DATE_FROM] + check_date_format(date_from) + if K_JSON_DATE_TO in data.keys(): + if data[K_JSON_DATE_TO] == "": + data_to = "*" + else: + date_to = data[K_JSON_DATE_TO] + check_date_format(date_to) + if K_JSON_SESSION in data.keys(): + session_id = data[K_JSON_SESSION] + + # Close json file and return + f.close() + return ( + study_id, + subjects, + session_id, + data_dict, + list_fars, + dcm2niix_path, + dcm2niix_opts, + date_from, + date_to, + ) + + +def generate_heuristic_file( + shanoir2bids_dict: object, path_heuristic_file: object +) -> None: + """Generate heudiconv heuristic.py file from shanoir2bids mapping dict + Parameters + ---------- + shanoir2bids_dict : + path_heuristic_file : path of the python heuristic file (.py) + """ + heuristic = f"""from heudiconv.heuristics.reproin import create_key + +def create_bids_key(dataset): + template = create_key(subdir=dataset['bidsDir'],file_suffix=dataset['bidsName'],outtype=("dicom","nii.gz")) + return template + +def get_dataset_to_key_mapping(shanoir2bids): + dataset_to_key = dict() + for dataset in shanoir2bids: + template = create_bids_key(dataset) + dataset_to_key[dataset['datasetName']] = template + return dataset_to_key + + +def infotodict(seqinfo): + + info = dict() + shanoir2bids = {shanoir2bids_dict} + + dataset_to_key = get_dataset_to_key_mapping(shanoir2bids) + for seq in seqinfo: + if seq.series_description in dataset_to_key.keys(): + key = dataset_to_key[seq.series_description] + if key in info.keys(): + info[key].append(seq.series_id) + else: + info[key] = [seq.series_id] + return info +""" + + with open(path_heuristic_file, "w", encoding="utf-8") as file: + file.write(heuristic) + file.close() + pass + + +class DownloadShanoirDatasetToBIDS: + """ + class that handles the downloading of shanoir data set and the reformatting as a BIDS data structure + """ + + def __init__(self): + """ + Initialize the class instance + """ + self.shanoir_subjects = None # List of Shanoir subjects + self.shanoir2bids_dict = ( + None # Dictionary specifying how to reformat data into BIDS structure + ) + self.shanoir_username = None # Shanoir username + self.shanoir_study_id = None # Shanoir study ID + self.shanoir_session_id = None # Shanoir study ID + self.shanoir_file_type = ( + DEFAULT_SHANOIR_FILE_TYPE # Default download type (nifti/dicom) + ) + self.json_config_file = None + self.list_fars = [] # List of substrings to edit in subjects names + self.dl_dir = None # download directory, where data will be stored + self.parser = None # Shanoir Downloader Parser + self.n_seq = 0 # Number of sequences in the shanoir2bids_dict + self.log_fn = None + self.dcm2niix_path = None # Path to the dcm2niix the user wants to use + self.dcm2niix_opts = None # Options to add to the dcm2niix call + self.dcm2niix_config_file = None # .json file to store dcm2niix options + self.date_from = None + self.date_to = None + self.longitudinal = False + self.to_automri_format = ( + False # Special filenames for automri (close to BIDS format) + ) + self.add_sns = False # Add series number suffix to filename + + def set_json_config_file(self, json_file): + """ + Sets the configuration for the download through a json file + :param json_file: str, path to the json_file + """ + self.json_config_file = json_file + ( + study_id, + subjects, + session_id, + data_dict, + list_fars, + dcm2niix_path, + dcm2niix_opts, + date_from, + date_to, + ) = read_json_config_file(json_file=json_file) + self.set_shanoir_study_id(study_id=study_id) + self.set_shanoir_subjects(subjects=subjects) + self.set_shanoir_session_id(session_id=session_id) + self.set_shanoir2bids_dict(data_dict=data_dict) + self.set_shanoir_list_find_and_replace(list_fars=list_fars) + self.set_dcm2niix_parameters( + dcm2niix_path=dcm2niix_path, dcm2niix_opts=dcm2niix_opts + ) + self.set_date_from(date_from=date_from) + self.set_date_to(date_to=date_to) + + def set_shanoir_file_type(self, shanoir_file_type): + if shanoir_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI]: + self.shanoir_file_type = shanoir_file_type + else: + sys.exit("Unknown shanoir file type {}".format(shanoir_file_type)) + + def set_shanoir_study_id(self, study_id): + self.shanoir_study_id = study_id + + def set_shanoir_username(self, shanoir_username): + self.shanoir_username = shanoir_username + + def set_shanoir_domaine(self, shanoir_domaine): + self.shanoir_domaine = shanoir_domaine + + def set_shanoir_subjects(self, subjects): + self.shanoir_subjects = subjects + + def set_shanoir_session_id(self, session_id): + self.shanoir_session_id = session_id + + def set_shanoir_list_find_and_replace(self, list_fars): + self.list_fars = list_fars + + def set_dcm2niix_parameters(self, dcm2niix_path, dcm2niix_opts): + self.dcm2niix_path = dcm2niix_path + self.dcm2niix_opts = dcm2niix_opts + + def set_dcm2niix_config_files(self, path_dcm2niix_options_files): + self.dcm2niix_config_file = path_dcm2niix_options_files + # Serializing json + json_object = json.dumps(self.dcm2niix_opts, indent=4) + with open(self.dcm2niix_config_file, "w") as file: + file.write(json_object) + + def set_date_from(self, date_from): + self.date_from = date_from + + def set_date_to(self, date_to): + self.date_to = date_to + + def set_shanoir2bids_dict(self, data_dict): + self.shanoir2bids_dict = data_dict + self.n_seq = len(self.shanoir2bids_dict) + + def set_download_directory(self, dl_dir): + if dl_dir is None: + # Create a default download directory + dt = datetime.datetime.now().strftime("%Y_%m_%d_at_%Hh%Mm%Ss") + self.dl_dir = "_".join( + ["shanoir2bids", "download", self.shanoir_study_id, dt] + ) + print( + "A NEW DEFAULT directory is created as you did not provide a download directory (-of option)\n\t" + + self.dl_dir + ) + else: + self.dl_dir = dl_dir + # Create directory if it does not exist + if not ope(self.dl_dir): + Path(self.dl_dir).mkdir(parents=True, exist_ok=True) + self.set_log_filename() + + def set_heuristic_file(self, path_heuristic_file): + if path_heuristic_file is None: + print("TO BE DONE") + else: + self.heuristic_file = path_heuristic_file + + def set_log_filename(self): + curr_time = datetime.datetime.now() + basename = "shanoir_downloader_{:04}{:02}{:02}_{:02}{:02}{:02}.log".format( + curr_time.year, + curr_time.month, + curr_time.day, + curr_time.hour, + curr_time.minute, + curr_time.second, + ) + self.log_fn = opj(self.dl_dir, basename) + + def toggle_longitudinal_version(self): + self.longitudinal = True + + def switch_to_automri_format(self): + self.to_automri_format = True + + def add_series_number_suffix(self): + self.add_sns = True + + def configure_parser(self): + """ + Configure the parser and the configuration of the shanoir_downloader + """ + self.parser = shanoir_downloader.create_arg_parser() + shanoir_downloader.add_common_arguments(self.parser) + shanoir_downloader.add_configuration_arguments(self.parser) + shanoir_downloader.add_search_arguments(self.parser) + shanoir_downloader.add_ids_arguments(self.parser) + + def download_subject(self, subject_to_search): + """ + For a single subject + 1. Downloads the Shanoir datasets + 2. Reorganises the Shanoir dataset as BIDS format as defined in the json configuration file provided by user + :param subject_to_search: + :return: + """ + banner_msg("Downloading subject " + subject_to_search) + + # Open log file to write the steps of processing (downloading, renaming...) + fp = open(self.log_fn, "a") + + # Real Shanoir2Bids mapping ( deal with search terms are included in datasetName field) + bids_mapping = [] + + # Loop on each sequence defined in the dictionary + for seq in range(self.n_seq): + # Isolate elements that are called many times + shanoir_seq_name = self.shanoir2bids_dict[seq][ + K_DS_NAME + ] # Shanoir sequence name (OLD) + bids_seq_subdir = self.shanoir2bids_dict[seq][ + K_BIDS_DIR + ] # Sequence BIDS subdirectory name (NEW) + bids_seq_name = self.shanoir2bids_dict[seq][ + K_BIDS_NAME + ] # Sequence BIDS nickname (NEW) + if self.longitudinal: + bids_seq_session = self.shanoir2bids_dict[seq][ + K_BIDS_SES + ] # Sequence BIDS nickname (NEW) + + # Print message concerning the sequence that is being downloaded + print( + "\t-", + bids_seq_name, + subject_to_search, + "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", + ) + + # Initialize the parser + search_txt = ( + "studyName:" + + self.shanoir_study_id.replace(" ", "?") + + " AND datasetName:" + + shanoir_seq_name.replace(" ", "?") + + " AND subjectName:" + + subject_to_search.replace(" ", "?") + + " AND examinationComment:" + + self.shanoir_session_id.replace(" ", "*") + + " AND examinationDate:[" + + self.date_from + + " TO " + + self.date_to + + "]" + ) + + args = self.parser.parse_args( + [ + "-u", + self.shanoir_username, + "-d", + self.shanoir_domaine, + "-of", + self.dl_dir, + "-em", + "-st", + search_txt, + "-s", + "200", + "-f", + self.shanoir_file_type, + "-so", + "id,ASC", + "-t", + "500", + ] + ) # Increase time out for heavy files + + config = shanoir_downloader.initialize(args) + response = shanoir_downloader.solr_search(config, args) + + # From response, process the data + # Print the number of items found and a list of these items + if response.status_code == 200: + # Invoke shanoir_downloader to download all the data + shanoir_downloader.download_search_results(config, args, response) + + if len(response.json()["content"]) == 0: + warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns +a result on the website. +Search Text : "{}" \n""".format( + search_txt + ) + print(warn_msg) + fp.write(warn_msg) + else: + for item in response.json()["content"]: + # Define subject_id + su_id = item["subjectName"] + # If the user has defined a list of edits to subject names... then do the find and replace + for far in self.list_fars: + su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) + + # ID of the subject (sub-*) + subject_id = su_id + # correct BIDS mapping of the searched dataset + bids_seq_mapping = { + "datasetName": item["datasetName"], + "bidsDir": bids_seq_subdir, + "bidsName": bids_seq_name, + "bids_subject_id": subject_id, + } + + if self.longitudinal: + bids_seq_mapping["bids_session_id"] = bids_seq_session + else: + bids_seq_mapping["bids_session_id"] = None + + bids_mapping.append(bids_seq_mapping) + + # Write the information on the data in the log file + fp.write("- datasetId = " + str(item["datasetId"]) + "\n") + fp.write(" -- studyName: " + item["studyName"] + "\n") + fp.write(" -- subjectName: " + item["subjectName"] + "\n") + fp.write(" -- session: " + item["examinationComment"] + "\n") + fp.write(" -- datasetName: " + item["datasetName"] + "\n") + fp.write( + " -- examinationDate: " + item["examinationDate"] + "\n" + ) + fp.write(" >> Downloading archive OK\n") + + # Create temp directory to make sure the directory is empty before + # TODO: Replace with temp directory ? + tmp_dir = opj(self.dl_dir, "temp_archive") + Path(tmp_dir).mkdir(parents=True, exist_ok=True) + + # Extract the downloaded archive + dl_archive = glob(opj(self.dl_dir, "*" + item["id"] + "*.zip"))[ + 0 + ] + with zipfile.ZipFile(dl_archive, "r") as zip_ref: + extraction_dir = opj(tmp_dir, item["id"]) + zip_ref.extractall(extraction_dir) + + fp.write( + " >> Extraction of all files from archive '" + + dl_archive + + " into " + + tmp_dir + + item["id"] + + "\n" + ) + + elif response.status_code == 204: + banner_msg("ERROR : No file found!") + fp.write(" >> ERROR : No file found!\n") + else: + banner_msg( + "ERROR : Returned by the request: status of the response = " + + response.status_code + ) + fp.write( + " >> ERROR : Returned by the request: status of the response = " + + str(response.status_code) + + "\n" + ) + # Generate Heudiconv heuristic file from .json mapping + generate_heuristic_file(bids_mapping, self.heuristic_file) + # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options + + if self.longitudinal: + workflow( + files=glob(opj(self.dl_dir, 'temp_archive', "*", "*.dcm"), recursive=True), + outdir=opj(self.dl_dir, "test"), + subjs=[subject_id], + session = bids_seq_session, + converter="dcm2niix", + heuristic=self.heuristic_file, + bids_options='--bids', + dcmconfig=self.dcm2niix_config_file, + datalad=True, + minmeta=True, + ) + else: + + workflow( + files=glob(opj(self.dl_dir,'temp_archive',"*", "*.dcm"), recursive=True), + outdir=opj(self.dl_dir, "test"), + subjs=[subject_id], + converter="dcm2niix", + heuristic=self.heuristic_file, + bids_options='--bids', + dcmconfig=self.dcm2niix_config_file, + datalad=True, + minmeta=True, + ) + fp.close() + + def download(self): + """ + Loop over the Shanoir subjects and go download the required datasets + :return: + """ + self.set_log_filename() + self.configure_parser() # Configure the shanoir_downloader parser + fp = open(self.log_fn, "w") + for subject_to_search in self.shanoir_subjects: + t_start_subject = time() + self.download_subject(subject_to_search=subject_to_search) + dur_min = int((time() - t_start_subject) // 60) + dur_sec = int((time() - t_start_subject) % 60) + end_msg = ( + "Downloaded dataset for subject " + + subject_to_search + + " in {}m{}s".format(dur_min, dur_sec) + ) + banner_msg(end_msg) + + +def main(): + # Parse argument for the script + parser = shanoir_downloader.create_arg_parser(description=DESCRIPTION) + # Use username and output folder arguments from shanoir_downloader + shanoir_downloader.add_username_argument(parser) + parser.add_argument( + "-d", + "--domain", + default="shanoir.irisa.fr", + help="The shanoir domain to query.", + ) + parser.add_argument( + "-f", + "--format", + default="dicom", + choices=["dicom"], + help="The format to download.", + ) + shanoir_downloader.add_output_folder_argument(parser=parser, required=False) + # Add the argument for the configuration file + parser.add_argument( + "-j", + "--config_file", + required=True, + help="Path to the .json configuration file specifying parameters for shanoir downloading.", + ) + parser.add_argument( + "-L", + "--longitudinal", + required=False, + action="store_true", + help="Toggle longitudinal approach.", + ) + # parser.add_argument( + # "-a", "--automri", action="store_true", help="Switch to automri file tree." + # ) + # parser.add_argument( + # "-A", + # "--add_sns", + # action="store_true", + # help="Add series number suffix (compatible with -a)", + # ) + # Parse arguments + args = parser.parse_args() + + # Start configuring the DownloadShanoirDatasetToBids class instance + stb = DownloadShanoirDatasetToBIDS() + stb.set_shanoir_username(args.username) + stb.set_shanoir_domaine(args.domain) + stb.set_json_config_file( + json_file=args.config_file + ) # path to json configuration file + stb.set_shanoir_file_type(shanoir_file_type=args.format) # Format (dicom or nifti) + stb.set_download_directory( + dl_dir=args.output_folder + ) # output folder (if None a default directory is created) + stb.set_heuristic_file(path_heuristic_file="/home/alpron/heuristic.py") + stb.set_dcm2niix_config_files( + path_dcm2niix_options_files="/home/alpron/dcm2niix_options.json" + ) + if args.longitudinal: + stb.toggle_longitudinal_version() + # if args.automri: + # stb.switch_to_automri_format() + # if args.add_sns: + # if not args.automri: + # print("Warning : -A option is only compatible with -a option.") + # stb.add_series_number_suffix() + + stb.download() + + +if __name__ == "__main__": + main() From b7384c673a9b79b8bcb20e346a7d3bed5b0e64ab Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 25 Mar 2024 11:48:34 +0100 Subject: [PATCH 25/86] - added temporary directories to download in DICOM archives and extracted and heuristic and tempfiles for dcm2niix configuration and heudivconv heuristic files. --- shanoir2bidsv1.py | 401 ++++++++++++++++++++++++---------------------- 1 file changed, 210 insertions(+), 191 deletions(-) diff --git a/shanoir2bidsv1.py b/shanoir2bidsv1.py index 7016425..e302238 100755 --- a/shanoir2bidsv1.py +++ b/shanoir2bidsv1.py @@ -6,20 +6,22 @@ # Script to download and BIDS-like organize data on Shanoir using "shanoir_downloader.py" developed by Arthur Masson # @Author: Malo Gaubert , Quentin Duché # @Date: 24 Juin 2022 + import os -import sys -import zipfile -import json -import shutil -import shanoir_downloader -from dotenv import load_dotenv -from pathlib import Path from os.path import join as opj, splitext as ops, exists as ope, dirname as opd from glob import glob +import sys +from pathlib import Path from time import time +import zipfile import datetime +import tempfile from dateutil import parser +import json +import shutil +import shanoir_downloader +from dotenv import load_dotenv from heudiconv.main import workflow @@ -238,7 +240,6 @@ def __init__(self): self.log_fn = None self.dcm2niix_path = None # Path to the dcm2niix the user wants to use self.dcm2niix_opts = None # Options to add to the dcm2niix call - self.dcm2niix_config_file = None # .json file to store dcm2niix options self.date_from = None self.date_to = None self.longitudinal = False @@ -303,11 +304,10 @@ def set_dcm2niix_parameters(self, dcm2niix_path, dcm2niix_opts): self.dcm2niix_path = dcm2niix_path self.dcm2niix_opts = dcm2niix_opts - def set_dcm2niix_config_files(self, path_dcm2niix_options_files): - self.dcm2niix_config_file = path_dcm2niix_options_files + def export_dcm2niix_config_options(self, path_dcm2niix_options_file): # Serializing json json_object = json.dumps(self.dcm2niix_opts, indent=4) - with open(self.dcm2niix_config_file, "w") as file: + with open(path_dcm2niix_options_file, "w") as file: file.write(json_object) def set_date_from(self, date_from): @@ -340,9 +340,15 @@ def set_download_directory(self, dl_dir): def set_heuristic_file(self, path_heuristic_file): if path_heuristic_file is None: - print("TO BE DONE") + print(f"No heuristic file provided") else: - self.heuristic_file = path_heuristic_file + filename, ext = ops(path_heuristic_file) + if ext != ".py": + print( + f"Provided heuristic file {path_heuristic_file} is not a .py file as expected" + ) + else: + self.heuristic_file = path_heuristic_file def set_log_filename(self): curr_time = datetime.datetime.now() @@ -388,192 +394,205 @@ def download_subject(self, subject_to_search): # Open log file to write the steps of processing (downloading, renaming...) fp = open(self.log_fn, "a") - # Real Shanoir2Bids mapping ( deal with search terms are included in datasetName field) + # Real Shanoir2Bids mapping (handle case when solr search term are included) bids_mapping = [] - # Loop on each sequence defined in the dictionary - for seq in range(self.n_seq): - # Isolate elements that are called many times - shanoir_seq_name = self.shanoir2bids_dict[seq][ - K_DS_NAME - ] # Shanoir sequence name (OLD) - bids_seq_subdir = self.shanoir2bids_dict[seq][ - K_BIDS_DIR - ] # Sequence BIDS subdirectory name (NEW) - bids_seq_name = self.shanoir2bids_dict[seq][ - K_BIDS_NAME - ] # Sequence BIDS nickname (NEW) - if self.longitudinal: - bids_seq_session = self.shanoir2bids_dict[seq][ - K_BIDS_SES - ] # Sequence BIDS nickname (NEW) - - # Print message concerning the sequence that is being downloaded - print( - "\t-", - bids_seq_name, - subject_to_search, - "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", - ) - - # Initialize the parser - search_txt = ( - "studyName:" - + self.shanoir_study_id.replace(" ", "?") - + " AND datasetName:" - + shanoir_seq_name.replace(" ", "?") - + " AND subjectName:" - + subject_to_search.replace(" ", "?") - + " AND examinationComment:" - + self.shanoir_session_id.replace(" ", "*") - + " AND examinationDate:[" - + self.date_from - + " TO " - + self.date_to - + "]" - ) - - args = self.parser.parse_args( - [ - "-u", - self.shanoir_username, - "-d", - self.shanoir_domaine, - "-of", - self.dl_dir, - "-em", - "-st", - search_txt, - "-s", - "200", - "-f", - self.shanoir_file_type, - "-so", - "id,ASC", - "-t", - "500", - ] - ) # Increase time out for heavy files - - config = shanoir_downloader.initialize(args) - response = shanoir_downloader.solr_search(config, args) - - # From response, process the data - # Print the number of items found and a list of these items - if response.status_code == 200: - # Invoke shanoir_downloader to download all the data - shanoir_downloader.download_search_results(config, args, response) - - if len(response.json()["content"]) == 0: - warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns -a result on the website. -Search Text : "{}" \n""".format( - search_txt + # temporary directory containing dowloaded DICOM.zip files + with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_dicom: + with tempfile.TemporaryDirectory( + dir=self.dl_dir + ) as tmp_archive: + print(tmp_archive) + # Loop on each sequence defined in the dictionary + for seq in range(self.n_seq): + # Isolate elements that are called many times + shanoir_seq_name = self.shanoir2bids_dict[seq][ + K_DS_NAME + ] # Shanoir sequence name (OLD) + bids_seq_subdir = self.shanoir2bids_dict[seq][ + K_BIDS_DIR + ] # Sequence BIDS subdirectory name (NEW) + bids_seq_name = self.shanoir2bids_dict[seq][ + K_BIDS_NAME + ] # Sequence BIDS nickname (NEW) + if self.longitudinal: + bids_seq_session = self.shanoir2bids_dict[seq][ + K_BIDS_SES + ] # Sequence BIDS nickname (NEW) + + # Print message concerning the sequence that is being downloaded + print( + "\t-", + bids_seq_name, + subject_to_search, + "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", ) - print(warn_msg) - fp.write(warn_msg) - else: - for item in response.json()["content"]: - # Define subject_id - su_id = item["subjectName"] - # If the user has defined a list of edits to subject names... then do the find and replace - for far in self.list_fars: - su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) - - # ID of the subject (sub-*) - subject_id = su_id - # correct BIDS mapping of the searched dataset - bids_seq_mapping = { - "datasetName": item["datasetName"], - "bidsDir": bids_seq_subdir, - "bidsName": bids_seq_name, - "bids_subject_id": subject_id, - } - - if self.longitudinal: - bids_seq_mapping["bids_session_id"] = bids_seq_session - else: - bids_seq_mapping["bids_session_id"] = None - bids_mapping.append(bids_seq_mapping) + # Initialize the parser + search_txt = ( + "studyName:" + + self.shanoir_study_id.replace(" ", "?") + + " AND datasetName:" + + shanoir_seq_name.replace(" ", "?") + + " AND subjectName:" + + subject_to_search.replace(" ", "?") + + " AND examinationComment:" + + self.shanoir_session_id.replace(" ", "*") + + " AND examinationDate:[" + + self.date_from + + " TO " + + self.date_to + + "]" + ) - # Write the information on the data in the log file - fp.write("- datasetId = " + str(item["datasetId"]) + "\n") - fp.write(" -- studyName: " + item["studyName"] + "\n") - fp.write(" -- subjectName: " + item["subjectName"] + "\n") - fp.write(" -- session: " + item["examinationComment"] + "\n") - fp.write(" -- datasetName: " + item["datasetName"] + "\n") - fp.write( - " -- examinationDate: " + item["examinationDate"] + "\n" - ) - fp.write(" >> Downloading archive OK\n") + args = self.parser.parse_args( + [ + "-u", + self.shanoir_username, + "-d", + self.shanoir_domaine, + "-of", + tmp_archive, + "-em", + "-st", + search_txt, + "-s", + "200", + "-f", + self.shanoir_file_type, + "-so", + "id,ASC", + "-t", + "500", + ] + ) # Increase time out for heavy files - # Create temp directory to make sure the directory is empty before - # TODO: Replace with temp directory ? - tmp_dir = opj(self.dl_dir, "temp_archive") - Path(tmp_dir).mkdir(parents=True, exist_ok=True) + config = shanoir_downloader.initialize(args) + response = shanoir_downloader.solr_search(config, args) - # Extract the downloaded archive - dl_archive = glob(opj(self.dl_dir, "*" + item["id"] + "*.zip"))[ - 0 - ] - with zipfile.ZipFile(dl_archive, "r") as zip_ref: - extraction_dir = opj(tmp_dir, item["id"]) - zip_ref.extractall(extraction_dir) + # From response, process the data + # Print the number of items found and a list of these items + if response.status_code == 200: + # Invoke shanoir_downloader to download all the data + shanoir_downloader.download_search_results( + config, args, response + ) + if len(response.json()["content"]) == 0: + warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns + a result on the website. + Search Text : "{}" \n""".format( + search_txt + ) + print(warn_msg) + fp.write(warn_msg) + else: + for item in response.json()["content"]: + # Define subject_id + su_id = item["subjectName"] + # If the user has defined a list of edits to subject names... then do the find and replace + for far in self.list_fars: + su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) + + # ID of the subject (sub-*) + subject_id = su_id + # correct BIDS mapping of the searched dataset + bids_seq_mapping = { + "datasetName": item["datasetName"], + "bidsDir": bids_seq_subdir, + "bidsName": bids_seq_name, + "bids_subject_id": subject_id, + } + + if self.longitudinal: + bids_seq_mapping[ + "bids_session_id" + ] = bids_seq_session + else: + bids_seq_mapping["bids_session_id"] = None + + bids_mapping.append(bids_seq_mapping) + + # Write the information on the data in the log file + fp.write( + "- datasetId = " + str(item["datasetId"]) + "\n" + ) + fp.write(" -- studyName: " + item["studyName"] + "\n") + fp.write( + " -- subjectName: " + item["subjectName"] + "\n" + ) + fp.write( + " -- session: " + item["examinationComment"] + "\n" + ) + fp.write( + " -- datasetName: " + item["datasetName"] + "\n" + ) + fp.write( + " -- examinationDate: " + + item["examinationDate"] + + "\n" + ) + fp.write(" >> Downloading archive OK\n") + + # Extract the downloaded archive + dl_archive = glob( + opj(tmp_archive, "*" + item["id"] + "*.zip") + )[0] + with zipfile.ZipFile(dl_archive, "r") as zip_ref: + extraction_dir = opj(tmp_dicom, item["id"]) + zip_ref.extractall(extraction_dir) + + fp.write( + " >> Extraction of all files from archive '" + + dl_archive + + " into " + + extraction_dir + + "\n" + ) + + elif response.status_code == 204: + banner_msg("ERROR : No file found!") + fp.write(" >> ERROR : No file found!\n") + else: + banner_msg( + "ERROR : Returned by the request: status of the response = " + + response.status_code + ) fp.write( - " >> Extraction of all files from archive '" - + dl_archive - + " into " - + tmp_dir - + item["id"] + " >> ERROR : Returned by the request: status of the response = " + + str(response.status_code) + "\n" ) - elif response.status_code == 204: - banner_msg("ERROR : No file found!") - fp.write(" >> ERROR : No file found!\n") - else: - banner_msg( - "ERROR : Returned by the request: status of the response = " - + response.status_code - ) - fp.write( - " >> ERROR : Returned by the request: status of the response = " - + str(response.status_code) - + "\n" - ) - # Generate Heudiconv heuristic file from .json mapping - generate_heuristic_file(bids_mapping, self.heuristic_file) - # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options - - if self.longitudinal: - workflow( - files=glob(opj(self.dl_dir, 'temp_archive', "*", "*.dcm"), recursive=True), - outdir=opj(self.dl_dir, "test"), - subjs=[subject_id], - session = bids_seq_session, - converter="dcm2niix", - heuristic=self.heuristic_file, - bids_options='--bids', - dcmconfig=self.dcm2niix_config_file, - datalad=True, - minmeta=True, - ) - else: - - workflow( - files=glob(opj(self.dl_dir,'temp_archive',"*", "*.dcm"), recursive=True), - outdir=opj(self.dl_dir, "test"), - subjs=[subject_id], - converter="dcm2niix", - heuristic=self.heuristic_file, - bids_options='--bids', - dcmconfig=self.dcm2niix_config_file, - datalad=True, - minmeta=True, - ) - fp.close() + # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options + with tempfile.NamedTemporaryFile(mode='r+', + encoding="utf-8", dir=self.dl_dir, suffix=".py" + ) as heuristic_file: + # Generate Heudiconv heuristic file from configuration.json mapping + generate_heuristic_file(bids_mapping, heuristic_file.name) + with tempfile.NamedTemporaryFile(mode='r+', + encoding="utf-8", dir=self.dl_dir, suffix=".json" + ) as dcm2niix_config_file: + self.export_dcm2niix_config_options(dcm2niix_config_file.name) + workflow_params = { + "files": glob( + opj(tmp_dicom, "*", "*.dcm"), recursive=True + ), + "outdir": opj(self.dl_dir, str(self.shanoir_study_id)), + "subjs": [subject_id], + "converter": "dcm2niix", + "heuristic": heuristic_file.name, + "bids_options": "--bids", + "dcmconfig": dcm2niix_config_file.name, + "datalad": True, + "minmeta": True, + } + + if self.longitudinal: + workflow_params["session"] = bids_seq_session + + workflow(**workflow_params) + fp.close() def download(self): """ @@ -652,10 +671,10 @@ def main(): stb.set_download_directory( dl_dir=args.output_folder ) # output folder (if None a default directory is created) - stb.set_heuristic_file(path_heuristic_file="/home/alpron/heuristic.py") - stb.set_dcm2niix_config_files( - path_dcm2niix_options_files="/home/alpron/dcm2niix_options.json" - ) + # stb.set_heuristic_file(path_heuristic_file="/home/alpron/heuristic.py") + # stb.set_dcm2niix_config_files( + # path_dcm2niix_options_files="/home/alpron/dcm2niix_options.json" + # ) if args.longitudinal: stb.toggle_longitudinal_version() # if args.automri: From b7a8b3f4a34f0fe02024ac1b2f304f667ad2231a Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 25 Mar 2024 14:11:16 +0100 Subject: [PATCH 26/86] - renamed few modality suffixes so that it fits BIDS spec --- s2b_example_config_EMISEP_long.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/s2b_example_config_EMISEP_long.json b/s2b_example_config_EMISEP_long.json index cf6f037..bc64ab7 100644 --- a/s2b_example_config_EMISEP_long.json +++ b/s2b_example_config_EMISEP_long.json @@ -4,8 +4,8 @@ "session": "08-74 MO ENCEPHALE", "data_to_bids": [ - {"datasetName": "3D T1 MPRAGE", "bidsDir": "brain", "bidsName": "t1w", "bidsSession": "M00"}, - {"datasetName": "*3d flair*", "bidsDir": "brain", "bidsName": "flair", "bidsSession": "M00"}, + {"datasetName": "3D T1 MPRAGE", "bidsDir": "brain", "bidsName": "T1w", "bidsSession": "M00"}, + {"datasetName": "*3d flair*", "bidsDir": "brain", "bidsName": "FLAIR", "bidsSession": "M00"}, {"datasetName": "*c1c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", "bidsSession": "M00"}, {"datasetName": "*c1-c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", "bidsSession": "M00"}, @@ -64,7 +64,7 @@ ], "dcm2niix":"~/softs/miniconda3/bin/dcm2niix", - "dcm2niix_options": {"verbose": true,"compress": "y", "anon_bids": false} + "dcm2niix_options": {"verbose": true,"compress": "y", "anon_bids": true} } From 52aa9cbfd3f6c961a7ae234d11a2e1bcd1dcddc8 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 26 Mar 2024 10:39:15 +0100 Subject: [PATCH 27/86] change main script name --- shanoir2bidsv1.py => shanoir2bids_heudiconv.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename shanoir2bidsv1.py => shanoir2bids_heudiconv.py (100%) diff --git a/shanoir2bidsv1.py b/shanoir2bids_heudiconv.py similarity index 100% rename from shanoir2bidsv1.py rename to shanoir2bids_heudiconv.py From ce3722aa705471e39b17383acbc5252eacce4fae Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Wed, 27 Mar 2024 13:58:41 +0100 Subject: [PATCH 28/86] + modified default heudiconv StudyID grouping (used all) to handle multiple StudyID per Examination + included run simplifications --- shanoir2bids_heudiconv.py | 41 ++++++++++++++++++++++++++------------- 1 file changed, 28 insertions(+), 13 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index e302238..dcb5b55 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -18,11 +18,14 @@ import tempfile from dateutil import parser import json +import logging import shutil import shanoir_downloader from dotenv import load_dotenv from heudiconv.main import workflow +# import loggger used in heudiconv workflow +from heudiconv.main import lgr # Load environment variables @@ -180,7 +183,8 @@ def generate_heuristic_file( heuristic = f"""from heudiconv.heuristics.reproin import create_key def create_bids_key(dataset): - template = create_key(subdir=dataset['bidsDir'],file_suffix=dataset['bidsName'],outtype=("dicom","nii.gz")) + + template = create_key(subdir=dataset['bidsDir'],file_suffix=r"run-{{item:02d}}_" + dataset['bidsName'],outtype=("dicom","nii.gz")) return template def get_dataset_to_key_mapping(shanoir2bids): @@ -190,6 +194,16 @@ def get_dataset_to_key_mapping(shanoir2bids): dataset_to_key[dataset['datasetName']] = template return dataset_to_key +def simplify_runs(info): + info_final = dict() + for key in info.keys(): + if len(info[key])==1: + new_template = key[0].replace('run-{{item:02d}}_','') + new_key = (new_template, key[1], key[2]) + info_final[new_key] = info[key] + else: + info_final[key] = info[key] + return info_final def infotodict(seqinfo): @@ -204,7 +218,9 @@ def infotodict(seqinfo): info[key].append(seq.series_id) else: info[key] = [seq.series_id] - return info + # remove run- key if not needed (one run only) + info_final = simplify_runs(info) + return info_final """ with open(path_heuristic_file, "w", encoding="utf-8") as file: @@ -396,12 +412,9 @@ def download_subject(self, subject_to_search): # Real Shanoir2Bids mapping (handle case when solr search term are included) bids_mapping = [] - # temporary directory containing dowloaded DICOM.zip files with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_dicom: - with tempfile.TemporaryDirectory( - dir=self.dl_dir - ) as tmp_archive: + with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_archive: print(tmp_archive) # Loop on each sequence defined in the dictionary for seq in range(self.n_seq): @@ -565,33 +578,35 @@ def download_subject(self, subject_to_search): ) # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options - with tempfile.NamedTemporaryFile(mode='r+', - encoding="utf-8", dir=self.dl_dir, suffix=".py" + with tempfile.NamedTemporaryFile( + mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" ) as heuristic_file: # Generate Heudiconv heuristic file from configuration.json mapping generate_heuristic_file(bids_mapping, heuristic_file.name) - with tempfile.NamedTemporaryFile(mode='r+', - encoding="utf-8", dir=self.dl_dir, suffix=".json" + with tempfile.NamedTemporaryFile( + mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" ) as dcm2niix_config_file: self.export_dcm2niix_config_options(dcm2niix_config_file.name) workflow_params = { - "files": glob( - opj(tmp_dicom, "*", "*.dcm"), recursive=True - ), + "files": glob(opj(tmp_dicom, "*", "*.dcm"), recursive=True), "outdir": opj(self.dl_dir, str(self.shanoir_study_id)), "subjs": [subject_id], "converter": "dcm2niix", "heuristic": heuristic_file.name, "bids_options": "--bids", + # "with_prov": True, "dcmconfig": dcm2niix_config_file.name, "datalad": True, "minmeta": True, + "grouping": "all", # other options are too restrictive (tested on EMISEP) } if self.longitudinal: workflow_params["session"] = bids_seq_session workflow(**workflow_params) + # TODO add nipype logging into shanoir log file ? + # TODO use provenance option ? currently not working properly fp.close() def download(self): From 12e1e430e8e6a53c3cd6e565c5f19a0cbc2cebef Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 25 Mar 2024 15:15:54 +0100 Subject: [PATCH 29/86] proposal on the s2b_example_config.json to enhance BIDS compatibility --- s2b_example_config.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 9333d6e..486e6cf 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -3,11 +3,11 @@ "subjects": ["VS_Aneravimm_010", "VS_Aneravimm_011"], "data_to_bids": [ - {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", "bidsName": "t1w-mprage"}, - {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "t2-hr"}, - {"datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", "bidsName": "ap-hipp"}, - {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "cusp66-ap-b3000"}, - {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "cusp66-ap-b0"} + {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", "bidsName": "_acq-mprage_T1w"}, + {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "_acq-hr_T2w"}, + {"datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", "bidsName": "_task-restingState_acq-hipp_dir-AP_bold"}, + {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b3000_dir-AP_dwi"}, + {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-AP_dwi"} ], "dcm2niix":"/home/qduche/Software/dcm2niix_lnx/dcm2niix", "dcm2niix_options": "-v 0 -z y", From e088703b5cd7a8f7ebb63fc2a4a89348c1f0e261 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 26 Mar 2024 12:36:34 +0100 Subject: [PATCH 30/86] fix underscores in filename suffixes --- s2b_example_config.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 486e6cf..3b568a0 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -3,8 +3,8 @@ "subjects": ["VS_Aneravimm_010", "VS_Aneravimm_011"], "data_to_bids": [ - {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", "bidsName": "_acq-mprage_T1w"}, - {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "_acq-hr_T2w"}, + {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", "bidsName": "acq-mprage_T1w"}, + {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "acq-hr_T2w"}, {"datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", "bidsName": "_task-restingState_acq-hipp_dir-AP_bold"}, {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b3000_dir-AP_dwi"}, {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-AP_dwi"} From 1992915991b6b5c2037ff682b48d3782636d1510 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 26 Mar 2024 12:36:54 +0100 Subject: [PATCH 31/86] fix underscores in filename suffixes --- shanoir2bids.py | 501 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 359 insertions(+), 142 deletions(-) diff --git a/shanoir2bids.py b/shanoir2bids.py index 50d751d..bda6bb2 100755 --- a/shanoir2bids.py +++ b/shanoir2bids.py @@ -25,15 +25,16 @@ # 1) Ajouter option pour conserver/supprimer le dicom folder apres conversion dcm2niix # Load environment variables -load_dotenv(dotenv_path=opj(opd(__file__), '.env')) +load_dotenv(dotenv_path=opj(opd(__file__), ".env")) + def banner_msg(msg): """ Print a message framed by a banner of "*" characters :param msg: """ - banner = '*' * (len(msg) + 6) - print(banner + '\n* ', msg, ' *\n' + banner) + banner = "*" * (len(msg) + 6) + print(banner + "\n* ", msg, " *\n" + banner) # Keys for json configuration file @@ -44,30 +45,40 @@ def banner_msg(msg): K_JSON_FIND_AND_REPLACE = "find_and_replace_subject" K_DCM2NIIX_PATH = "dcm2niix" K_DCM2NIIX_OPTS = "dcm2niix_options" -K_FIND = 'find' -K_REPLACE = 'replace' -K_JSON_DATE_FROM = 'date_from' # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] -K_JSON_DATE_TO = 'date_to' # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] +K_FIND = "find" +K_REPLACE = "replace" +K_JSON_DATE_FROM = ( + "date_from" # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] +) +K_JSON_DATE_TO = ( + "date_to" # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] +) LIST_MANDATORY_KEYS_JSON = [K_JSON_STUDY_NAME, K_JSON_L_SUBJECTS, K_JSON_DATA_DICT] -LIST_AUTHORIZED_KEYS_JSON = LIST_MANDATORY_KEYS_JSON + [K_DCM2NIIX_PATH, K_DCM2NIIX_OPTS, K_JSON_DATE_FROM, K_JSON_DATE_TO, K_JSON_SESSION] +LIST_AUTHORIZED_KEYS_JSON = LIST_MANDATORY_KEYS_JSON + [ + K_DCM2NIIX_PATH, + K_DCM2NIIX_OPTS, + K_JSON_DATE_FROM, + K_JSON_DATE_TO, + K_JSON_SESSION, +] # Define keys for data dictionary -K_BIDS_NAME = 'bidsName' -K_BIDS_DIR = 'bidsDir' -K_BIDS_SES = 'bidsSession' -K_DS_NAME = 'datasetName' +K_BIDS_NAME = "bidsName" +K_BIDS_DIR = "bidsDir" +K_BIDS_SES = "bidsSession" +K_DS_NAME = "datasetName" # Define Extensions that are dealt so far by (#todo : think of other possible extensions ?) -NIFTI = '.nii' -NIIGZ = '.nii.gz' -JSON = '.json' -BVAL = '.bval' -BVEC = '.bvec' -DCM = '.dcm' +NIFTI = ".nii" +NIIGZ = ".nii.gz" +JSON = ".json" +BVAL = ".bval" +BVEC = ".bvec" +DCM = ".dcm" # Shanoir parameters -SHANOIR_FILE_TYPE_NIFTI = 'nifti' -SHANOIR_FILE_TYPE_DICOM = 'dicom' +SHANOIR_FILE_TYPE_NIFTI = "nifti" +SHANOIR_FILE_TYPE_DICOM = "dicom" DEFAULT_SHANOIR_FILE_TYPE = SHANOIR_FILE_TYPE_NIFTI # Define error and warning messages when call to dcm2niix is not well configured in the json file @@ -85,7 +96,10 @@ def check_date_format(date_to_format): parser.parse(date_to_format) # If the date validation goes wrong except ValueError: - print("Incorrect data format, should be YYYY-MM-DDTHH:MM:SSZ (for example: 2020-02-19T00:00:00Z)") + print( + "Incorrect data format, should be YYYY-MM-DDTHH:MM:SSZ (for example: 2020-02-19T00:00:00Z)" + ) + def read_json_config_file(json_file): """ @@ -113,9 +127,9 @@ def read_json_config_file(json_file): list_fars = [] dcm2niix_path = None dcm2niix_opts = None - date_from = '*' - date_to = '*' - session_id = '*' + date_from = "*" + date_to = "*" + session_id = "*" if K_JSON_FIND_AND_REPLACE in data.keys(): list_fars = data[K_JSON_FIND_AND_REPLACE] @@ -124,40 +138,54 @@ def read_json_config_file(json_file): if K_DCM2NIIX_OPTS in data.keys(): dcm2niix_opts = data[K_DCM2NIIX_OPTS] if K_JSON_DATE_FROM in data.keys(): - if data[K_JSON_DATE_FROM] == '': - data_from = '*' + if data[K_JSON_DATE_FROM] == "": + data_from = "*" else: date_from = data[K_JSON_DATE_FROM] check_date_format(date_from) if K_JSON_DATE_TO in data.keys(): - if data[K_JSON_DATE_TO] == '': - data_to = '*' + if data[K_JSON_DATE_TO] == "": + data_to = "*" else: date_to = data[K_JSON_DATE_TO] check_date_format(date_to) if K_JSON_SESSION in data.keys(): session_id = data[K_JSON_SESSION] - # Close json file and return f.close() - return study_id, subjects, session_id, data_dict, list_fars, dcm2niix_path, dcm2niix_opts, date_from, date_to + return ( + study_id, + subjects, + session_id, + data_dict, + list_fars, + dcm2niix_path, + dcm2niix_opts, + date_from, + date_to, + ) class DownloadShanoirDatasetToBIDS: """ class that handles the downloading of shanoir data set and the reformatting as a BIDS data structure """ + def __init__(self): """ Initialize the class instance """ self.shanoir_subjects = None # List of Shanoir subjects - self.shanoir2bids_dict = None # Dictionary specifying how to reformat data into BIDS structure + self.shanoir2bids_dict = ( + None # Dictionary specifying how to reformat data into BIDS structure + ) self.shanoir_username = None # Shanoir username self.shanoir_study_id = None # Shanoir study ID self.shanoir_session_id = None # Shanoir study ID - self.shanoir_file_type = DEFAULT_SHANOIR_FILE_TYPE # Default download type (nifti/dicom) + self.shanoir_file_type = ( + DEFAULT_SHANOIR_FILE_TYPE # Default download type (nifti/dicom) + ) self.json_config_file = None self.list_fars = [] # List of substrings to edit in subjects names self.dl_dir = None # download directory, where data will be stored @@ -169,8 +197,10 @@ def __init__(self): self.date_from = None self.date_to = None self.longitudinal = False - self.to_automri_format = False # Special filenames for automri (close to BIDS format) - self.add_sns = False # Add series number suffix to filename + self.to_automri_format = ( + False # Special filenames for automri (close to BIDS format) + ) + self.add_sns = False # Add series number suffix to filename def set_json_config_file(self, json_file): """ @@ -178,13 +208,25 @@ def set_json_config_file(self, json_file): :param json_file: str, path to the json_file """ self.json_config_file = json_file - study_id, subjects, session_id, data_dict, list_fars, dcm2niix_path, dcm2niix_opts, date_from, date_to = read_json_config_file(json_file=json_file) + ( + study_id, + subjects, + session_id, + data_dict, + list_fars, + dcm2niix_path, + dcm2niix_opts, + date_from, + date_to, + ) = read_json_config_file(json_file=json_file) self.set_shanoir_study_id(study_id=study_id) self.set_shanoir_subjects(subjects=subjects) self.set_shanoir_session_id(session_id=session_id) self.set_shanoir2bids_dict(data_dict=data_dict) self.set_shanoir_list_find_and_replace(list_fars=list_fars) - self.set_dcm2niix_parameters(dcm2niix_path=dcm2niix_path, dcm2niix_opts=dcm2niix_opts) + self.set_dcm2niix_parameters( + dcm2niix_path=dcm2niix_path, dcm2niix_opts=dcm2niix_opts + ) self.set_date_from(date_from=date_from) self.set_date_to(date_to=date_to) @@ -192,7 +234,7 @@ def set_shanoir_file_type(self, shanoir_file_type): if shanoir_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI]: self.shanoir_file_type = shanoir_file_type else: - sys.exit('Unknown shanoir file type {}'.format(shanoir_file_type)) + sys.exit("Unknown shanoir file type {}".format(shanoir_file_type)) def set_shanoir_study_id(self, study_id): self.shanoir_study_id = study_id @@ -200,7 +242,7 @@ def set_shanoir_study_id(self, study_id): def set_shanoir_username(self, shanoir_username): self.shanoir_username = shanoir_username - def set_shanoir_domaine(self, shanoir_domaine): + def set_shanoir_domaine(self, shanoir_domaine): self.shanoir_domaine = shanoir_domaine def set_shanoir_subjects(self, subjects): @@ -230,8 +272,13 @@ def set_download_directory(self, dl_dir): if dl_dir is None: # Create a default download directory dt = datetime.datetime.now().strftime("%Y_%m_%d_at_%Hh%Mm%Ss") - self.dl_dir = '_'.join(["shanoir2bids", "download", self.shanoir_study_id, dt]) - print('A NEW DEFAULT directory is created as you did not provide a download directory (-of option)\n\t' + self.dl_dir) + self.dl_dir = "_".join( + ["shanoir2bids", "download", self.shanoir_study_id, dt] + ) + print( + "A NEW DEFAULT directory is created as you did not provide a download directory (-of option)\n\t" + + self.dl_dir + ) else: self.dl_dir = dl_dir # Create directory if it does not exist @@ -241,12 +288,14 @@ def set_download_directory(self, dl_dir): def set_log_filename(self): curr_time = datetime.datetime.now() - basename = 'shanoir_downloader_{:04}{:02}{:02}_{:02}{:02}{:02}.log'.format(curr_time.year, - curr_time.month, - curr_time.day, - curr_time.hour, - curr_time.minute, - curr_time.second) + basename = "shanoir_downloader_{:04}{:02}{:02}_{:02}{:02}{:02}.log".format( + curr_time.year, + curr_time.month, + curr_time.day, + curr_time.hour, + curr_time.minute, + curr_time.second, + ) self.log_fn = opj(self.dl_dir, basename) def toggle_longitudinal_version(self): @@ -279,37 +328,71 @@ def download_subject(self, subject_to_search): banner_msg("Downloading subject " + subject_to_search) # Open log file to write the steps of processing (downloading, renaming...) - fp = open(self.log_fn, 'a') + fp = open(self.log_fn, "a") # Loop on each sequence defined in the dictionary for seq in range(self.n_seq): # Isolate elements that are called many times - shanoir_seq_name = self.shanoir2bids_dict[seq][K_DS_NAME] # Shanoir sequence name (OLD) - bids_seq_subdir = self.shanoir2bids_dict[seq][K_BIDS_DIR] # Sequence BIDS subdirectory name (NEW) - bids_seq_name = self.shanoir2bids_dict[seq][K_BIDS_NAME] # Sequence BIDS nickname (NEW) + shanoir_seq_name = self.shanoir2bids_dict[seq][ + K_DS_NAME + ] # Shanoir sequence name (OLD) + bids_seq_subdir = self.shanoir2bids_dict[seq][ + K_BIDS_DIR + ] # Sequence BIDS subdirectory name (NEW) + bids_seq_name = self.shanoir2bids_dict[seq][ + K_BIDS_NAME + ] # Sequence BIDS nickname (NEW) if self.longitudinal: - bids_seq_session = self.shanoir2bids_dict[seq][K_BIDS_SES] # Sequence BIDS nickname (NEW) + bids_seq_session = self.shanoir2bids_dict[seq][ + K_BIDS_SES + ] # Sequence BIDS nickname (NEW) # Print message concerning the sequence that is being downloaded - print('\t-', bids_seq_name, subject_to_search, '[' + str(seq + 1) + '/' + str(self.n_seq) + ']') + print( + "\t-", + bids_seq_name, + subject_to_search, + "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", + ) # Initialize the parser - search_txt = 'studyName:' + self.shanoir_study_id.replace(" ", "?") + \ - ' AND datasetName:' + shanoir_seq_name.replace(" ", "?") + \ - ' AND subjectName:' + subject_to_search.replace(" ", "?") + \ - ' AND examinationComment:' + self.shanoir_session_id.replace(" ", "*") + \ - ' AND examinationDate:[' + self.date_from + ' TO ' + self.date_to + ']' + search_txt = ( + "studyName:" + + self.shanoir_study_id.replace(" ", "?") + + " AND datasetName:" + + shanoir_seq_name.replace(" ", "?") + + " AND subjectName:" + + subject_to_search.replace(" ", "?") + + " AND examinationComment:" + + self.shanoir_session_id.replace(" ", "*") + + " AND examinationDate:[" + + self.date_from + + " TO " + + self.date_to + + "]" + ) args = self.parser.parse_args( - ['-u', self.shanoir_username, - '-d', self.shanoir_domaine, - '-of', self.dl_dir, - '-em', - '-st', search_txt, - '-s', '200', - '-f', self.shanoir_file_type, - '-so', 'id,ASC', - '-t', '500']) # Increase time out for heavy files + [ + "-u", + self.shanoir_username, + "-d", + self.shanoir_domaine, + "-of", + self.dl_dir, + "-em", + "-st", + search_txt, + "-s", + "200", + "-f", + self.shanoir_file_type, + "-so", + "id,ASC", + "-t", + "500", + ] + ) # Increase time out for heavy files config = shanoir_downloader.initialize(args) response = shanoir_downloader.solr_search(config, args) @@ -317,60 +400,71 @@ def download_subject(self, subject_to_search): # From response, process the data # Print the number of items found and a list of these items if response.status_code == 200: - # Invoke shanoir_downloader to download all the data shanoir_downloader.download_search_results(config, args, response) - if len(response.json()['content']) == 0: + if len(response.json()["content"]) == 0: warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns a result on the website. -Search Text : "{}" \n""".format(search_txt) +Search Text : "{}" \n""".format( + search_txt + ) print(warn_msg) fp.write(warn_msg) else: # Organize in BIDS like specifications and rename files - for item in response.json()['content']: + for item in response.json()["content"]: # Define subject_id - su_id = item['subjectName'] + su_id = item["subjectName"] # If the user has defined a list of edits to subject names... then do the find and replace for far in self.list_fars: su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) # ID of the subject (sub-*) - subject_id = 'sub-' + su_id + subject_id = "sub-" + su_id # Write the information on the data in the log file - fp.write('- datasetId = ' + str(item['datasetId']) + '\n') - fp.write(' -- studyName: ' + item['studyName'] + '\n') - fp.write(' -- subjectName: ' + item['subjectName'] + '\n') - fp.write(' -- session: ' + item['examinationComment'] + '\n') - fp.write(' -- datasetName: ' + item['datasetName'] + '\n') - fp.write(' -- examinationDate: ' + item['examinationDate'] + '\n') - fp.write(' >> Downloading archive OK\n') + fp.write("- datasetId = " + str(item["datasetId"]) + "\n") + fp.write(" -- studyName: " + item["studyName"] + "\n") + fp.write(" -- subjectName: " + item["subjectName"] + "\n") + fp.write(" -- session: " + item["examinationComment"] + "\n") + fp.write(" -- datasetName: " + item["datasetName"] + "\n") + fp.write( + " -- examinationDate: " + item["examinationDate"] + "\n" + ) + fp.write(" >> Downloading archive OK\n") # Subject BIDS directory if self.to_automri_format: - subject_dir = opj(self.dl_dir, 'su_' + su_id) + subject_dir = opj(self.dl_dir, "su_" + su_id) else: subject_dir = opj(self.dl_dir, subject_id) # Prepare BIDS naming if self.longitudinal: # Insert a session sub-directory - bids_data_dir = opj(subject_dir, bids_seq_session, bids_seq_subdir) + bids_data_dir = opj( + subject_dir, bids_seq_session, bids_seq_subdir + ) if self.to_automri_format: - bids_data_basename = '_'.join([bids_seq_session, bids_seq_name]) + bids_data_basename = "_".join( + [bids_seq_session, bids_seq_name] + ) else: - bids_data_basename = '_'.join([subject_id, bids_seq_session, bids_seq_name]) + bids_data_basename = "_".join( + [subject_id, bids_seq_session, bids_seq_name] + ) else: bids_data_dir = opj(subject_dir, bids_seq_subdir) if self.to_automri_format: - bids_data_basename = '_'.join([bids_seq_name, su_id]) + bids_data_basename = "_".join([bids_seq_name, su_id]) else: - bids_data_basename = '_'.join([subject_id, bids_seq_name]) + bids_data_basename = "_".join( + [subject_id, bids_seq_name] + ) # Create temp directory to make sure the directory is empty before - tmp_dir = opj(self.dl_dir, 'temp_archive') + tmp_dir = opj(self.dl_dir, "temp_archive") Path(tmp_dir).mkdir(parents=True, exist_ok=True) # Create the directory of the subject @@ -379,13 +473,21 @@ def download_subject(self, subject_to_search): Path(bids_data_dir).mkdir(parents=True, exist_ok=True) # Extract the downloaded archive - dl_archive = glob(opj(self.dl_dir, '*' + item['id'] + '*.zip'))[0] - with zipfile.ZipFile(dl_archive, 'r') as zip_ref: + dl_archive = glob(opj(self.dl_dir, "*" + item["id"] + "*.zip"))[ + 0 + ] + with zipfile.ZipFile(dl_archive, "r") as zip_ref: zip_ref.extractall(tmp_dir) # Get the list of files in the archive - list_unzipped_files = glob(opj(tmp_dir, '*')) + list_unzipped_files = glob(opj(tmp_dir, "*")) - fp.write(" >> Extraction of all files from archive '" + dl_archive + " into " + tmp_dir + "\n") + fp.write( + " >> Extraction of all files from archive '" + + dl_archive + + " into " + + tmp_dir + + "\n" + ) def seq_name(extension, run_num=0): """ @@ -394,12 +496,16 @@ def seq_name(extension, run_num=0): run_num : int, if 0 no suffix, else adds suffix '_run-1' or '_run-2' etc... """ if run_num > 0: - basename = bids_data_basename + '_run-{0}{1}'.format(run_num, extension) + basename = bids_data_basename + "_run-{0}{1}".format( + run_num, extension + ) else: basename = bids_data_basename + extension return opj(bids_data_dir, basename) - def check_duplicated_data(list_existing_file_json, file_to_add_json): + def check_duplicated_data( + list_existing_file_json, file_to_add_json + ): """ For a list of json file, check if a json file already exists meaning. We check the flags AcquisitionTime and SequenceName to derive equality test. @@ -414,7 +520,10 @@ def check_duplicated_data(list_existing_file_json, file_to_add_json): f_old = open(old_json) data_old = json.load(f_old) - if (data['AcquisitionTime'] == data_old['AcquisitionTime']) & (data['SequenceName'] == data_old['SequenceName']): + if ( + data["AcquisitionTime"] + == data_old["AcquisitionTime"] + ) & (data["SequenceName"] == data_old["SequenceName"]): # If one of the json file has the same AcquisitionTime and SequenceName, then it is a duplicated file f_old.close() f.close() @@ -426,29 +535,41 @@ def check_duplicated_data(list_existing_file_json, file_to_add_json): if self.shanoir_file_type == SHANOIR_FILE_TYPE_DICOM: # Process the DICOM file by calling dcm2niix if not self.dcm2niix_path: - err_msg = "msg\n- {file}""".format(msg=DCM2NIIX_ERR_MSG, file=self.json_config_file) + err_msg = "msg\n- {file}" "".format( + msg=DCM2NIIX_ERR_MSG, file=self.json_config_file + ) sys.exit(err_msg) if not self.dcm2niix_opts: - warn_msg = "msg\n- {file}""".format(msg=DCM2NIIX_WARN_MSG, file=self.json_config_file) + warn_msg = "msg\n- {file}" "".format( + msg=DCM2NIIX_WARN_MSG, file=self.json_config_file + ) print(warn_msg) - dcm_files = glob(opj(tmp_dir, '*' + DCM)) + dcm_files = glob(opj(tmp_dir, "*" + DCM)) # Define dcm2niix options - options = ' {opts} -f {basename} -o {out}'.format(opts=self.dcm2niix_opts, basename=bids_data_basename, out=tmp_dir) - cmd = self.dcm2niix_path + options + ' ' + dcm_files[0] + options = " {opts} -f {basename} -o {out}".format( + opts=self.dcm2niix_opts, + basename=bids_data_basename, + out=tmp_dir, + ) + cmd = self.dcm2niix_path + options + " " + dcm_files[0] # Retrieve dcm2niix output and save it to file info_dcm = os.popen(cmd) info_dcm = info_dcm.read() - info_dcm = info_dcm.split('\n') - fp.write('[dcm2niix] ' + '\n[dcm2niix] '.join(info_dcm[2:-1]) + '\n') + info_dcm = info_dcm.split("\n") + fp.write( + "[dcm2niix] " + + "\n[dcm2niix] ".join(info_dcm[2:-1]) + + "\n" + ) # Remove temporary DICOM files for dcm_file in dcm_files: os.remove(dcm_file) # After the command, the user should have a nifti and json file and extra files - list_unzipped_files = glob(opj(tmp_dir, '*')) + list_unzipped_files = glob(opj(tmp_dir, "*")) # Now the DICOM part of the script should be in the same stage as the NIFTI part of the script which is below @@ -468,13 +589,17 @@ def check_duplicated_data(list_existing_file_json, file_to_add_json): duplicated_data = False if self.to_automri_format and self.add_sns: - # Update bids_data_basename with Series Number information - for filename in list_unzipped_files: - if filename.endswith(JSON): - f_temp = open(filename) - json_dataset = json.load(f_temp) - series_number_suffix = str(json_dataset['SeriesNumber']) - bids_data_basename = '_'.join([bids_seq_name, su_id, series_number_suffix]) + # Update bids_data_basename with Series Number information + for filename in list_unzipped_files: + if filename.endswith(JSON): + f_temp = open(filename) + json_dataset = json.load(f_temp) + series_number_suffix = str( + json_dataset["SeriesNumber"] + ) + bids_data_basename = "_".join( + [bids_seq_name, su_id, series_number_suffix] + ) # Rename every element in the list of files that was in the archive for f in list_unzipped_files: # Loop over files in the archive @@ -488,13 +613,15 @@ def check_duplicated_data(list_existing_file_json, file_to_add_json): # And update variables. Filename as now '.nii.gz' extension f = filename + NIIGZ ext = NIIGZ - if ext == '.gz': + if ext == ".gz": # os.path.splitext returns (filename.nii, '.gz') instead of (filename, '.nii.gz') ext = NIIGZ # Let's process and rename the file # Compare the contents of the associated json file between previous and new file "AcquisitionTime" - list_existing_f_ext = glob(opj(bids_data_dir, bids_data_basename + '*' + ext)) + list_existing_f_ext = glob( + opj(bids_data_dir, bids_data_basename + "*" + ext) + ) nf = len(list_existing_f_ext) # if files with same bids_data_basename are present in subjects directory, check json files @@ -502,11 +629,20 @@ def check_duplicated_data(list_existing_file_json, file_to_add_json): for filename_tempo in list_unzipped_files: fn_tempo, ext_tempo = ops(filename_tempo) if (ext_tempo == JSON) & (ext == JSON): - list_existing_f_ext_json = glob(opj(bids_data_dir, bids_data_basename + '*json')) - duplicated_data = check_duplicated_data(list_existing_f_ext_json, filename_tempo) + list_existing_f_ext_json = glob( + opj( + bids_data_dir, + bids_data_basename + "*json", + ) + ) + duplicated_data = check_duplicated_data( + list_existing_f_ext_json, filename_tempo + ) if duplicated_data: - fp.write(" \n/!\ File already present in the subject's directory. Data will not be used.\n\n") + fp.write( + " \n/!\ File already present in the subject's directory. Data will not be used.\n\n" + ) list_unzipped_files = [] if ext in [NIIGZ, JSON, BVAL, BVEC]: @@ -515,45 +651,89 @@ def check_duplicated_data(list_existing_file_json, file_to_add_json): if nf == 0: # No previously existing file : perform the renaming os.rename(f, bids_filename) - fp.write(" >> Renaming to '" + bids_filename + "'\n") + fp.write( + " >> Renaming to '" + + bids_filename + + "'\n" + ) elif nf == 1: - fp.write(' /!\ One similar filename found ! \n') + fp.write( + " /!\ One similar filename found ! \n" + ) # One file already existed : give suffices # the old file gets run-1 suffix - os.rename(bids_filename, seq_name(extension=ext, run_num=1)) - fp.write(" >> Renaming '" + bids_filename + "' to '" + seq_name(extension=ext, run_num=1) + "'\n") + os.rename( + bids_filename, + seq_name(extension=ext, run_num=1), + ) + fp.write( + " >> Renaming '" + + bids_filename + + "' to '" + + seq_name(extension=ext, run_num=1) + + "'\n" + ) # the new file gets run-2 suffix os.rename(f, seq_name(extension=ext, run_num=2)) - fp.write(" >> Renaming to '" + seq_name(extension=ext, run_num=2) + "'\n") + fp.write( + " >> Renaming to '" + + seq_name(extension=ext, run_num=2) + + "'\n" + ) else: # nf >= 2 # At least two files exist, do not touch previous but add the right suffix to new file - os.rename(f, seq_name(extension=ext, run_num=nf + 1)) - fp.write(" >> Renaming to '" + seq_name(extension=ext, run_num=nf + 1) + "'\n") + os.rename( + f, seq_name(extension=ext, run_num=nf + 1) + ) + fp.write( + " >> Renaming to '" + + seq_name(extension=ext, run_num=nf + 1) + + "'\n" + ) else: - print('[BIDS format] The extension', ext, 'is not yet dealt. Ask the authors of the script to make an effort.') - fp.write(' >> [BIDS format] The extension' + ext + 'is not yet dealt. Ask the authors of the script to make an effort.\n') + print( + "[BIDS format] The extension", + ext, + "is not yet dealt. Ask the authors of the script to make an effort.", + ) + fp.write( + " >> [BIDS format] The extension" + + ext + + "is not yet dealt. Ask the authors of the script to make an effort.\n" + ) # Delete temporary directory (remove files before if duplicated) if duplicated_data: for f in list_unzipped_files: filename, ext = ops(f) if ext == NIFTI: - if os.path.exists(f + '.gz'): os.remove(f + '.gz') + if os.path.exists(f + ".gz"): + os.remove(f + ".gz") else: - if os.path.exists(f): os.remove(f) + if os.path.exists(f): + os.remove(f) shutil.rmtree(tmp_dir) - fp.write(' >> Deleting temporary dir ' + tmp_dir + '\n') + fp.write(" >> Deleting temporary dir " + tmp_dir + "\n") os.remove(dl_archive) - fp.write(' >> Deleting downloaded archive ' + dl_archive + '\n\n\n') + fp.write( + " >> Deleting downloaded archive " + dl_archive + "\n\n\n" + ) # End for item in response.json() # End else (cas ou response.json()['content']) != 0) elif response.status_code == 204: - banner_msg('ERROR : No file found!') - fp.write(' >> ERROR : No file found!\n') + banner_msg("ERROR : No file found!") + fp.write(" >> ERROR : No file found!\n") else: - banner_msg('ERROR : Returned by the request: status of the response = ' + response.status_code) - fp.write(' >> ERROR : Returned by the request: status of the response = ' + str(response.status_code) + '\n') + banner_msg( + "ERROR : Returned by the request: status of the response = " + + response.status_code + ) + fp.write( + " >> ERROR : Returned by the request: status of the response = " + + str(response.status_code) + + "\n" + ) fp.close() @@ -564,13 +744,17 @@ def download(self): """ self.set_log_filename() self.configure_parser() # Configure the shanoir_downloader parser - fp = open(self.log_fn, 'w') + fp = open(self.log_fn, "w") for subject_to_search in self.shanoir_subjects: t_start_subject = time() self.download_subject(subject_to_search=subject_to_search) dur_min = int((time() - t_start_subject) // 60) dur_sec = int((time() - t_start_subject) % 60) - end_msg = 'Downloaded dataset for subject ' + subject_to_search + ' in {}m{}s'.format(dur_min, dur_sec) + end_msg = ( + "Downloaded dataset for subject " + + subject_to_search + + " in {}m{}s".format(dur_min, dur_sec) + ) banner_msg(end_msg) @@ -579,14 +763,43 @@ def main(): parser = shanoir_downloader.create_arg_parser(description=DESCRIPTION) # Use username and output folder arguments from shanoir_downloader shanoir_downloader.add_username_argument(parser) - parser.add_argument('-d', '--domain', default='shanoir.irisa.fr', help='The shanoir domain to query.') - parser.add_argument('-f', '--format', default='nifti', choices=['nifti', 'dicom'], help='The format to download.') + parser.add_argument( + "-d", + "--domain", + default="shanoir.irisa.fr", + help="The shanoir domain to query.", + ) + parser.add_argument( + "-f", + "--format", + default="nifti", + choices=["nifti", "dicom"], + help="The format to download.", + ) shanoir_downloader.add_output_folder_argument(parser=parser, required=False) # Add the argument for the configuration file - parser.add_argument('-j', '--config_file', required=True, help='Path to the .json configuration file specifying parameters for shanoir downloading.') - parser.add_argument('-L', '--longitudinal', required=False, action='store_true', help='Toggle longitudinal approach.') - parser.add_argument('-a', '--automri', action='store_true', help='Switch to automri file tree.') - parser.add_argument('-A', '--add_sns', action='store_true', help='Add series number suffix (compatible with -a)') + parser.add_argument( + "-j", + "--config_file", + required=True, + help="Path to the .json configuration file specifying parameters for shanoir downloading.", + ) + parser.add_argument( + "-L", + "--longitudinal", + required=False, + action="store_true", + help="Toggle longitudinal approach.", + ) + parser.add_argument( + "-a", "--automri", action="store_true", help="Switch to automri file tree." + ) + parser.add_argument( + "-A", + "--add_sns", + action="store_true", + help="Add series number suffix (compatible with -a)", + ) # Parse arguments args = parser.parse_args() @@ -594,19 +807,23 @@ def main(): stb = DownloadShanoirDatasetToBIDS() stb.set_shanoir_username(args.username) stb.set_shanoir_domaine(args.domain) - stb.set_json_config_file(json_file=args.config_file) # path to json configuration file + stb.set_json_config_file( + json_file=args.config_file + ) # path to json configuration file stb.set_shanoir_file_type(shanoir_file_type=args.format) # Format (dicom or nifti) - stb.set_download_directory(dl_dir=args.output_folder) # output folder (if None a default directory is created) + stb.set_download_directory( + dl_dir=args.output_folder + ) # output folder (if None a default directory is created) if args.longitudinal: stb.toggle_longitudinal_version() if args.automri: stb.switch_to_automri_format() if args.add_sns: if not args.automri: - print('Warning : -A option is only compatible with -a option.') + print("Warning : -A option is only compatible with -a option.") stb.add_series_number_suffix() stb.download() -if __name__ == '__main__': +if __name__ == "__main__": main() From 4afe579726f723275a2328cb8b8b75c51df5a9a3 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 26 Mar 2024 14:23:47 +0100 Subject: [PATCH 32/86] Revert "fix underscores in filename suffixes" This reverts commit eaf442a1f3d46a1903aa15809ef5b6933f75bb59. --- shanoir2bids.py | 501 ++++++++++++++---------------------------------- 1 file changed, 142 insertions(+), 359 deletions(-) diff --git a/shanoir2bids.py b/shanoir2bids.py index bda6bb2..50d751d 100755 --- a/shanoir2bids.py +++ b/shanoir2bids.py @@ -25,16 +25,15 @@ # 1) Ajouter option pour conserver/supprimer le dicom folder apres conversion dcm2niix # Load environment variables -load_dotenv(dotenv_path=opj(opd(__file__), ".env")) - +load_dotenv(dotenv_path=opj(opd(__file__), '.env')) def banner_msg(msg): """ Print a message framed by a banner of "*" characters :param msg: """ - banner = "*" * (len(msg) + 6) - print(banner + "\n* ", msg, " *\n" + banner) + banner = '*' * (len(msg) + 6) + print(banner + '\n* ', msg, ' *\n' + banner) # Keys for json configuration file @@ -45,40 +44,30 @@ def banner_msg(msg): K_JSON_FIND_AND_REPLACE = "find_and_replace_subject" K_DCM2NIIX_PATH = "dcm2niix" K_DCM2NIIX_OPTS = "dcm2niix_options" -K_FIND = "find" -K_REPLACE = "replace" -K_JSON_DATE_FROM = ( - "date_from" # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] -) -K_JSON_DATE_TO = ( - "date_to" # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] -) +K_FIND = 'find' +K_REPLACE = 'replace' +K_JSON_DATE_FROM = 'date_from' # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] +K_JSON_DATE_TO = 'date_to' # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] LIST_MANDATORY_KEYS_JSON = [K_JSON_STUDY_NAME, K_JSON_L_SUBJECTS, K_JSON_DATA_DICT] -LIST_AUTHORIZED_KEYS_JSON = LIST_MANDATORY_KEYS_JSON + [ - K_DCM2NIIX_PATH, - K_DCM2NIIX_OPTS, - K_JSON_DATE_FROM, - K_JSON_DATE_TO, - K_JSON_SESSION, -] +LIST_AUTHORIZED_KEYS_JSON = LIST_MANDATORY_KEYS_JSON + [K_DCM2NIIX_PATH, K_DCM2NIIX_OPTS, K_JSON_DATE_FROM, K_JSON_DATE_TO, K_JSON_SESSION] # Define keys for data dictionary -K_BIDS_NAME = "bidsName" -K_BIDS_DIR = "bidsDir" -K_BIDS_SES = "bidsSession" -K_DS_NAME = "datasetName" +K_BIDS_NAME = 'bidsName' +K_BIDS_DIR = 'bidsDir' +K_BIDS_SES = 'bidsSession' +K_DS_NAME = 'datasetName' # Define Extensions that are dealt so far by (#todo : think of other possible extensions ?) -NIFTI = ".nii" -NIIGZ = ".nii.gz" -JSON = ".json" -BVAL = ".bval" -BVEC = ".bvec" -DCM = ".dcm" +NIFTI = '.nii' +NIIGZ = '.nii.gz' +JSON = '.json' +BVAL = '.bval' +BVEC = '.bvec' +DCM = '.dcm' # Shanoir parameters -SHANOIR_FILE_TYPE_NIFTI = "nifti" -SHANOIR_FILE_TYPE_DICOM = "dicom" +SHANOIR_FILE_TYPE_NIFTI = 'nifti' +SHANOIR_FILE_TYPE_DICOM = 'dicom' DEFAULT_SHANOIR_FILE_TYPE = SHANOIR_FILE_TYPE_NIFTI # Define error and warning messages when call to dcm2niix is not well configured in the json file @@ -96,10 +85,7 @@ def check_date_format(date_to_format): parser.parse(date_to_format) # If the date validation goes wrong except ValueError: - print( - "Incorrect data format, should be YYYY-MM-DDTHH:MM:SSZ (for example: 2020-02-19T00:00:00Z)" - ) - + print("Incorrect data format, should be YYYY-MM-DDTHH:MM:SSZ (for example: 2020-02-19T00:00:00Z)") def read_json_config_file(json_file): """ @@ -127,9 +113,9 @@ def read_json_config_file(json_file): list_fars = [] dcm2niix_path = None dcm2niix_opts = None - date_from = "*" - date_to = "*" - session_id = "*" + date_from = '*' + date_to = '*' + session_id = '*' if K_JSON_FIND_AND_REPLACE in data.keys(): list_fars = data[K_JSON_FIND_AND_REPLACE] @@ -138,54 +124,40 @@ def read_json_config_file(json_file): if K_DCM2NIIX_OPTS in data.keys(): dcm2niix_opts = data[K_DCM2NIIX_OPTS] if K_JSON_DATE_FROM in data.keys(): - if data[K_JSON_DATE_FROM] == "": - data_from = "*" + if data[K_JSON_DATE_FROM] == '': + data_from = '*' else: date_from = data[K_JSON_DATE_FROM] check_date_format(date_from) if K_JSON_DATE_TO in data.keys(): - if data[K_JSON_DATE_TO] == "": - data_to = "*" + if data[K_JSON_DATE_TO] == '': + data_to = '*' else: date_to = data[K_JSON_DATE_TO] check_date_format(date_to) if K_JSON_SESSION in data.keys(): session_id = data[K_JSON_SESSION] + # Close json file and return f.close() - return ( - study_id, - subjects, - session_id, - data_dict, - list_fars, - dcm2niix_path, - dcm2niix_opts, - date_from, - date_to, - ) + return study_id, subjects, session_id, data_dict, list_fars, dcm2niix_path, dcm2niix_opts, date_from, date_to class DownloadShanoirDatasetToBIDS: """ class that handles the downloading of shanoir data set and the reformatting as a BIDS data structure """ - def __init__(self): """ Initialize the class instance """ self.shanoir_subjects = None # List of Shanoir subjects - self.shanoir2bids_dict = ( - None # Dictionary specifying how to reformat data into BIDS structure - ) + self.shanoir2bids_dict = None # Dictionary specifying how to reformat data into BIDS structure self.shanoir_username = None # Shanoir username self.shanoir_study_id = None # Shanoir study ID self.shanoir_session_id = None # Shanoir study ID - self.shanoir_file_type = ( - DEFAULT_SHANOIR_FILE_TYPE # Default download type (nifti/dicom) - ) + self.shanoir_file_type = DEFAULT_SHANOIR_FILE_TYPE # Default download type (nifti/dicom) self.json_config_file = None self.list_fars = [] # List of substrings to edit in subjects names self.dl_dir = None # download directory, where data will be stored @@ -197,10 +169,8 @@ def __init__(self): self.date_from = None self.date_to = None self.longitudinal = False - self.to_automri_format = ( - False # Special filenames for automri (close to BIDS format) - ) - self.add_sns = False # Add series number suffix to filename + self.to_automri_format = False # Special filenames for automri (close to BIDS format) + self.add_sns = False # Add series number suffix to filename def set_json_config_file(self, json_file): """ @@ -208,25 +178,13 @@ def set_json_config_file(self, json_file): :param json_file: str, path to the json_file """ self.json_config_file = json_file - ( - study_id, - subjects, - session_id, - data_dict, - list_fars, - dcm2niix_path, - dcm2niix_opts, - date_from, - date_to, - ) = read_json_config_file(json_file=json_file) + study_id, subjects, session_id, data_dict, list_fars, dcm2niix_path, dcm2niix_opts, date_from, date_to = read_json_config_file(json_file=json_file) self.set_shanoir_study_id(study_id=study_id) self.set_shanoir_subjects(subjects=subjects) self.set_shanoir_session_id(session_id=session_id) self.set_shanoir2bids_dict(data_dict=data_dict) self.set_shanoir_list_find_and_replace(list_fars=list_fars) - self.set_dcm2niix_parameters( - dcm2niix_path=dcm2niix_path, dcm2niix_opts=dcm2niix_opts - ) + self.set_dcm2niix_parameters(dcm2niix_path=dcm2niix_path, dcm2niix_opts=dcm2niix_opts) self.set_date_from(date_from=date_from) self.set_date_to(date_to=date_to) @@ -234,7 +192,7 @@ def set_shanoir_file_type(self, shanoir_file_type): if shanoir_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI]: self.shanoir_file_type = shanoir_file_type else: - sys.exit("Unknown shanoir file type {}".format(shanoir_file_type)) + sys.exit('Unknown shanoir file type {}'.format(shanoir_file_type)) def set_shanoir_study_id(self, study_id): self.shanoir_study_id = study_id @@ -242,7 +200,7 @@ def set_shanoir_study_id(self, study_id): def set_shanoir_username(self, shanoir_username): self.shanoir_username = shanoir_username - def set_shanoir_domaine(self, shanoir_domaine): + def set_shanoir_domaine(self, shanoir_domaine): self.shanoir_domaine = shanoir_domaine def set_shanoir_subjects(self, subjects): @@ -272,13 +230,8 @@ def set_download_directory(self, dl_dir): if dl_dir is None: # Create a default download directory dt = datetime.datetime.now().strftime("%Y_%m_%d_at_%Hh%Mm%Ss") - self.dl_dir = "_".join( - ["shanoir2bids", "download", self.shanoir_study_id, dt] - ) - print( - "A NEW DEFAULT directory is created as you did not provide a download directory (-of option)\n\t" - + self.dl_dir - ) + self.dl_dir = '_'.join(["shanoir2bids", "download", self.shanoir_study_id, dt]) + print('A NEW DEFAULT directory is created as you did not provide a download directory (-of option)\n\t' + self.dl_dir) else: self.dl_dir = dl_dir # Create directory if it does not exist @@ -288,14 +241,12 @@ def set_download_directory(self, dl_dir): def set_log_filename(self): curr_time = datetime.datetime.now() - basename = "shanoir_downloader_{:04}{:02}{:02}_{:02}{:02}{:02}.log".format( - curr_time.year, - curr_time.month, - curr_time.day, - curr_time.hour, - curr_time.minute, - curr_time.second, - ) + basename = 'shanoir_downloader_{:04}{:02}{:02}_{:02}{:02}{:02}.log'.format(curr_time.year, + curr_time.month, + curr_time.day, + curr_time.hour, + curr_time.minute, + curr_time.second) self.log_fn = opj(self.dl_dir, basename) def toggle_longitudinal_version(self): @@ -328,71 +279,37 @@ def download_subject(self, subject_to_search): banner_msg("Downloading subject " + subject_to_search) # Open log file to write the steps of processing (downloading, renaming...) - fp = open(self.log_fn, "a") + fp = open(self.log_fn, 'a') # Loop on each sequence defined in the dictionary for seq in range(self.n_seq): # Isolate elements that are called many times - shanoir_seq_name = self.shanoir2bids_dict[seq][ - K_DS_NAME - ] # Shanoir sequence name (OLD) - bids_seq_subdir = self.shanoir2bids_dict[seq][ - K_BIDS_DIR - ] # Sequence BIDS subdirectory name (NEW) - bids_seq_name = self.shanoir2bids_dict[seq][ - K_BIDS_NAME - ] # Sequence BIDS nickname (NEW) + shanoir_seq_name = self.shanoir2bids_dict[seq][K_DS_NAME] # Shanoir sequence name (OLD) + bids_seq_subdir = self.shanoir2bids_dict[seq][K_BIDS_DIR] # Sequence BIDS subdirectory name (NEW) + bids_seq_name = self.shanoir2bids_dict[seq][K_BIDS_NAME] # Sequence BIDS nickname (NEW) if self.longitudinal: - bids_seq_session = self.shanoir2bids_dict[seq][ - K_BIDS_SES - ] # Sequence BIDS nickname (NEW) + bids_seq_session = self.shanoir2bids_dict[seq][K_BIDS_SES] # Sequence BIDS nickname (NEW) # Print message concerning the sequence that is being downloaded - print( - "\t-", - bids_seq_name, - subject_to_search, - "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", - ) + print('\t-', bids_seq_name, subject_to_search, '[' + str(seq + 1) + '/' + str(self.n_seq) + ']') # Initialize the parser - search_txt = ( - "studyName:" - + self.shanoir_study_id.replace(" ", "?") - + " AND datasetName:" - + shanoir_seq_name.replace(" ", "?") - + " AND subjectName:" - + subject_to_search.replace(" ", "?") - + " AND examinationComment:" - + self.shanoir_session_id.replace(" ", "*") - + " AND examinationDate:[" - + self.date_from - + " TO " - + self.date_to - + "]" - ) + search_txt = 'studyName:' + self.shanoir_study_id.replace(" ", "?") + \ + ' AND datasetName:' + shanoir_seq_name.replace(" ", "?") + \ + ' AND subjectName:' + subject_to_search.replace(" ", "?") + \ + ' AND examinationComment:' + self.shanoir_session_id.replace(" ", "*") + \ + ' AND examinationDate:[' + self.date_from + ' TO ' + self.date_to + ']' args = self.parser.parse_args( - [ - "-u", - self.shanoir_username, - "-d", - self.shanoir_domaine, - "-of", - self.dl_dir, - "-em", - "-st", - search_txt, - "-s", - "200", - "-f", - self.shanoir_file_type, - "-so", - "id,ASC", - "-t", - "500", - ] - ) # Increase time out for heavy files + ['-u', self.shanoir_username, + '-d', self.shanoir_domaine, + '-of', self.dl_dir, + '-em', + '-st', search_txt, + '-s', '200', + '-f', self.shanoir_file_type, + '-so', 'id,ASC', + '-t', '500']) # Increase time out for heavy files config = shanoir_downloader.initialize(args) response = shanoir_downloader.solr_search(config, args) @@ -400,71 +317,60 @@ def download_subject(self, subject_to_search): # From response, process the data # Print the number of items found and a list of these items if response.status_code == 200: + # Invoke shanoir_downloader to download all the data shanoir_downloader.download_search_results(config, args, response) - if len(response.json()["content"]) == 0: + if len(response.json()['content']) == 0: warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns a result on the website. -Search Text : "{}" \n""".format( - search_txt - ) +Search Text : "{}" \n""".format(search_txt) print(warn_msg) fp.write(warn_msg) else: # Organize in BIDS like specifications and rename files - for item in response.json()["content"]: + for item in response.json()['content']: # Define subject_id - su_id = item["subjectName"] + su_id = item['subjectName'] # If the user has defined a list of edits to subject names... then do the find and replace for far in self.list_fars: su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) # ID of the subject (sub-*) - subject_id = "sub-" + su_id + subject_id = 'sub-' + su_id # Write the information on the data in the log file - fp.write("- datasetId = " + str(item["datasetId"]) + "\n") - fp.write(" -- studyName: " + item["studyName"] + "\n") - fp.write(" -- subjectName: " + item["subjectName"] + "\n") - fp.write(" -- session: " + item["examinationComment"] + "\n") - fp.write(" -- datasetName: " + item["datasetName"] + "\n") - fp.write( - " -- examinationDate: " + item["examinationDate"] + "\n" - ) - fp.write(" >> Downloading archive OK\n") + fp.write('- datasetId = ' + str(item['datasetId']) + '\n') + fp.write(' -- studyName: ' + item['studyName'] + '\n') + fp.write(' -- subjectName: ' + item['subjectName'] + '\n') + fp.write(' -- session: ' + item['examinationComment'] + '\n') + fp.write(' -- datasetName: ' + item['datasetName'] + '\n') + fp.write(' -- examinationDate: ' + item['examinationDate'] + '\n') + fp.write(' >> Downloading archive OK\n') # Subject BIDS directory if self.to_automri_format: - subject_dir = opj(self.dl_dir, "su_" + su_id) + subject_dir = opj(self.dl_dir, 'su_' + su_id) else: subject_dir = opj(self.dl_dir, subject_id) # Prepare BIDS naming if self.longitudinal: # Insert a session sub-directory - bids_data_dir = opj( - subject_dir, bids_seq_session, bids_seq_subdir - ) + bids_data_dir = opj(subject_dir, bids_seq_session, bids_seq_subdir) if self.to_automri_format: - bids_data_basename = "_".join( - [bids_seq_session, bids_seq_name] - ) + bids_data_basename = '_'.join([bids_seq_session, bids_seq_name]) else: - bids_data_basename = "_".join( - [subject_id, bids_seq_session, bids_seq_name] - ) + bids_data_basename = '_'.join([subject_id, bids_seq_session, bids_seq_name]) else: bids_data_dir = opj(subject_dir, bids_seq_subdir) if self.to_automri_format: - bids_data_basename = "_".join([bids_seq_name, su_id]) + bids_data_basename = '_'.join([bids_seq_name, su_id]) else: - bids_data_basename = "_".join( - [subject_id, bids_seq_name] - ) + bids_data_basename = '_'.join([subject_id, bids_seq_name]) # Create temp directory to make sure the directory is empty before - tmp_dir = opj(self.dl_dir, "temp_archive") + tmp_dir = opj(self.dl_dir, 'temp_archive') Path(tmp_dir).mkdir(parents=True, exist_ok=True) # Create the directory of the subject @@ -473,21 +379,13 @@ def download_subject(self, subject_to_search): Path(bids_data_dir).mkdir(parents=True, exist_ok=True) # Extract the downloaded archive - dl_archive = glob(opj(self.dl_dir, "*" + item["id"] + "*.zip"))[ - 0 - ] - with zipfile.ZipFile(dl_archive, "r") as zip_ref: + dl_archive = glob(opj(self.dl_dir, '*' + item['id'] + '*.zip'))[0] + with zipfile.ZipFile(dl_archive, 'r') as zip_ref: zip_ref.extractall(tmp_dir) # Get the list of files in the archive - list_unzipped_files = glob(opj(tmp_dir, "*")) + list_unzipped_files = glob(opj(tmp_dir, '*')) - fp.write( - " >> Extraction of all files from archive '" - + dl_archive - + " into " - + tmp_dir - + "\n" - ) + fp.write(" >> Extraction of all files from archive '" + dl_archive + " into " + tmp_dir + "\n") def seq_name(extension, run_num=0): """ @@ -496,16 +394,12 @@ def seq_name(extension, run_num=0): run_num : int, if 0 no suffix, else adds suffix '_run-1' or '_run-2' etc... """ if run_num > 0: - basename = bids_data_basename + "_run-{0}{1}".format( - run_num, extension - ) + basename = bids_data_basename + '_run-{0}{1}'.format(run_num, extension) else: basename = bids_data_basename + extension return opj(bids_data_dir, basename) - def check_duplicated_data( - list_existing_file_json, file_to_add_json - ): + def check_duplicated_data(list_existing_file_json, file_to_add_json): """ For a list of json file, check if a json file already exists meaning. We check the flags AcquisitionTime and SequenceName to derive equality test. @@ -520,10 +414,7 @@ def check_duplicated_data( f_old = open(old_json) data_old = json.load(f_old) - if ( - data["AcquisitionTime"] - == data_old["AcquisitionTime"] - ) & (data["SequenceName"] == data_old["SequenceName"]): + if (data['AcquisitionTime'] == data_old['AcquisitionTime']) & (data['SequenceName'] == data_old['SequenceName']): # If one of the json file has the same AcquisitionTime and SequenceName, then it is a duplicated file f_old.close() f.close() @@ -535,41 +426,29 @@ def check_duplicated_data( if self.shanoir_file_type == SHANOIR_FILE_TYPE_DICOM: # Process the DICOM file by calling dcm2niix if not self.dcm2niix_path: - err_msg = "msg\n- {file}" "".format( - msg=DCM2NIIX_ERR_MSG, file=self.json_config_file - ) + err_msg = "msg\n- {file}""".format(msg=DCM2NIIX_ERR_MSG, file=self.json_config_file) sys.exit(err_msg) if not self.dcm2niix_opts: - warn_msg = "msg\n- {file}" "".format( - msg=DCM2NIIX_WARN_MSG, file=self.json_config_file - ) + warn_msg = "msg\n- {file}""".format(msg=DCM2NIIX_WARN_MSG, file=self.json_config_file) print(warn_msg) - dcm_files = glob(opj(tmp_dir, "*" + DCM)) + dcm_files = glob(opj(tmp_dir, '*' + DCM)) # Define dcm2niix options - options = " {opts} -f {basename} -o {out}".format( - opts=self.dcm2niix_opts, - basename=bids_data_basename, - out=tmp_dir, - ) - cmd = self.dcm2niix_path + options + " " + dcm_files[0] + options = ' {opts} -f {basename} -o {out}'.format(opts=self.dcm2niix_opts, basename=bids_data_basename, out=tmp_dir) + cmd = self.dcm2niix_path + options + ' ' + dcm_files[0] # Retrieve dcm2niix output and save it to file info_dcm = os.popen(cmd) info_dcm = info_dcm.read() - info_dcm = info_dcm.split("\n") - fp.write( - "[dcm2niix] " - + "\n[dcm2niix] ".join(info_dcm[2:-1]) - + "\n" - ) + info_dcm = info_dcm.split('\n') + fp.write('[dcm2niix] ' + '\n[dcm2niix] '.join(info_dcm[2:-1]) + '\n') # Remove temporary DICOM files for dcm_file in dcm_files: os.remove(dcm_file) # After the command, the user should have a nifti and json file and extra files - list_unzipped_files = glob(opj(tmp_dir, "*")) + list_unzipped_files = glob(opj(tmp_dir, '*')) # Now the DICOM part of the script should be in the same stage as the NIFTI part of the script which is below @@ -589,17 +468,13 @@ def check_duplicated_data( duplicated_data = False if self.to_automri_format and self.add_sns: - # Update bids_data_basename with Series Number information - for filename in list_unzipped_files: - if filename.endswith(JSON): - f_temp = open(filename) - json_dataset = json.load(f_temp) - series_number_suffix = str( - json_dataset["SeriesNumber"] - ) - bids_data_basename = "_".join( - [bids_seq_name, su_id, series_number_suffix] - ) + # Update bids_data_basename with Series Number information + for filename in list_unzipped_files: + if filename.endswith(JSON): + f_temp = open(filename) + json_dataset = json.load(f_temp) + series_number_suffix = str(json_dataset['SeriesNumber']) + bids_data_basename = '_'.join([bids_seq_name, su_id, series_number_suffix]) # Rename every element in the list of files that was in the archive for f in list_unzipped_files: # Loop over files in the archive @@ -613,15 +488,13 @@ def check_duplicated_data( # And update variables. Filename as now '.nii.gz' extension f = filename + NIIGZ ext = NIIGZ - if ext == ".gz": + if ext == '.gz': # os.path.splitext returns (filename.nii, '.gz') instead of (filename, '.nii.gz') ext = NIIGZ # Let's process and rename the file # Compare the contents of the associated json file between previous and new file "AcquisitionTime" - list_existing_f_ext = glob( - opj(bids_data_dir, bids_data_basename + "*" + ext) - ) + list_existing_f_ext = glob(opj(bids_data_dir, bids_data_basename + '*' + ext)) nf = len(list_existing_f_ext) # if files with same bids_data_basename are present in subjects directory, check json files @@ -629,20 +502,11 @@ def check_duplicated_data( for filename_tempo in list_unzipped_files: fn_tempo, ext_tempo = ops(filename_tempo) if (ext_tempo == JSON) & (ext == JSON): - list_existing_f_ext_json = glob( - opj( - bids_data_dir, - bids_data_basename + "*json", - ) - ) - duplicated_data = check_duplicated_data( - list_existing_f_ext_json, filename_tempo - ) + list_existing_f_ext_json = glob(opj(bids_data_dir, bids_data_basename + '*json')) + duplicated_data = check_duplicated_data(list_existing_f_ext_json, filename_tempo) if duplicated_data: - fp.write( - " \n/!\ File already present in the subject's directory. Data will not be used.\n\n" - ) + fp.write(" \n/!\ File already present in the subject's directory. Data will not be used.\n\n") list_unzipped_files = [] if ext in [NIIGZ, JSON, BVAL, BVEC]: @@ -651,89 +515,45 @@ def check_duplicated_data( if nf == 0: # No previously existing file : perform the renaming os.rename(f, bids_filename) - fp.write( - " >> Renaming to '" - + bids_filename - + "'\n" - ) + fp.write(" >> Renaming to '" + bids_filename + "'\n") elif nf == 1: - fp.write( - " /!\ One similar filename found ! \n" - ) + fp.write(' /!\ One similar filename found ! \n') # One file already existed : give suffices # the old file gets run-1 suffix - os.rename( - bids_filename, - seq_name(extension=ext, run_num=1), - ) - fp.write( - " >> Renaming '" - + bids_filename - + "' to '" - + seq_name(extension=ext, run_num=1) - + "'\n" - ) + os.rename(bids_filename, seq_name(extension=ext, run_num=1)) + fp.write(" >> Renaming '" + bids_filename + "' to '" + seq_name(extension=ext, run_num=1) + "'\n") # the new file gets run-2 suffix os.rename(f, seq_name(extension=ext, run_num=2)) - fp.write( - " >> Renaming to '" - + seq_name(extension=ext, run_num=2) - + "'\n" - ) + fp.write(" >> Renaming to '" + seq_name(extension=ext, run_num=2) + "'\n") else: # nf >= 2 # At least two files exist, do not touch previous but add the right suffix to new file - os.rename( - f, seq_name(extension=ext, run_num=nf + 1) - ) - fp.write( - " >> Renaming to '" - + seq_name(extension=ext, run_num=nf + 1) - + "'\n" - ) + os.rename(f, seq_name(extension=ext, run_num=nf + 1)) + fp.write(" >> Renaming to '" + seq_name(extension=ext, run_num=nf + 1) + "'\n") else: - print( - "[BIDS format] The extension", - ext, - "is not yet dealt. Ask the authors of the script to make an effort.", - ) - fp.write( - " >> [BIDS format] The extension" - + ext - + "is not yet dealt. Ask the authors of the script to make an effort.\n" - ) + print('[BIDS format] The extension', ext, 'is not yet dealt. Ask the authors of the script to make an effort.') + fp.write(' >> [BIDS format] The extension' + ext + 'is not yet dealt. Ask the authors of the script to make an effort.\n') # Delete temporary directory (remove files before if duplicated) if duplicated_data: for f in list_unzipped_files: filename, ext = ops(f) if ext == NIFTI: - if os.path.exists(f + ".gz"): - os.remove(f + ".gz") + if os.path.exists(f + '.gz'): os.remove(f + '.gz') else: - if os.path.exists(f): - os.remove(f) + if os.path.exists(f): os.remove(f) shutil.rmtree(tmp_dir) - fp.write(" >> Deleting temporary dir " + tmp_dir + "\n") + fp.write(' >> Deleting temporary dir ' + tmp_dir + '\n') os.remove(dl_archive) - fp.write( - " >> Deleting downloaded archive " + dl_archive + "\n\n\n" - ) + fp.write(' >> Deleting downloaded archive ' + dl_archive + '\n\n\n') # End for item in response.json() # End else (cas ou response.json()['content']) != 0) elif response.status_code == 204: - banner_msg("ERROR : No file found!") - fp.write(" >> ERROR : No file found!\n") + banner_msg('ERROR : No file found!') + fp.write(' >> ERROR : No file found!\n') else: - banner_msg( - "ERROR : Returned by the request: status of the response = " - + response.status_code - ) - fp.write( - " >> ERROR : Returned by the request: status of the response = " - + str(response.status_code) - + "\n" - ) + banner_msg('ERROR : Returned by the request: status of the response = ' + response.status_code) + fp.write(' >> ERROR : Returned by the request: status of the response = ' + str(response.status_code) + '\n') fp.close() @@ -744,17 +564,13 @@ def download(self): """ self.set_log_filename() self.configure_parser() # Configure the shanoir_downloader parser - fp = open(self.log_fn, "w") + fp = open(self.log_fn, 'w') for subject_to_search in self.shanoir_subjects: t_start_subject = time() self.download_subject(subject_to_search=subject_to_search) dur_min = int((time() - t_start_subject) // 60) dur_sec = int((time() - t_start_subject) % 60) - end_msg = ( - "Downloaded dataset for subject " - + subject_to_search - + " in {}m{}s".format(dur_min, dur_sec) - ) + end_msg = 'Downloaded dataset for subject ' + subject_to_search + ' in {}m{}s'.format(dur_min, dur_sec) banner_msg(end_msg) @@ -763,43 +579,14 @@ def main(): parser = shanoir_downloader.create_arg_parser(description=DESCRIPTION) # Use username and output folder arguments from shanoir_downloader shanoir_downloader.add_username_argument(parser) - parser.add_argument( - "-d", - "--domain", - default="shanoir.irisa.fr", - help="The shanoir domain to query.", - ) - parser.add_argument( - "-f", - "--format", - default="nifti", - choices=["nifti", "dicom"], - help="The format to download.", - ) + parser.add_argument('-d', '--domain', default='shanoir.irisa.fr', help='The shanoir domain to query.') + parser.add_argument('-f', '--format', default='nifti', choices=['nifti', 'dicom'], help='The format to download.') shanoir_downloader.add_output_folder_argument(parser=parser, required=False) # Add the argument for the configuration file - parser.add_argument( - "-j", - "--config_file", - required=True, - help="Path to the .json configuration file specifying parameters for shanoir downloading.", - ) - parser.add_argument( - "-L", - "--longitudinal", - required=False, - action="store_true", - help="Toggle longitudinal approach.", - ) - parser.add_argument( - "-a", "--automri", action="store_true", help="Switch to automri file tree." - ) - parser.add_argument( - "-A", - "--add_sns", - action="store_true", - help="Add series number suffix (compatible with -a)", - ) + parser.add_argument('-j', '--config_file', required=True, help='Path to the .json configuration file specifying parameters for shanoir downloading.') + parser.add_argument('-L', '--longitudinal', required=False, action='store_true', help='Toggle longitudinal approach.') + parser.add_argument('-a', '--automri', action='store_true', help='Switch to automri file tree.') + parser.add_argument('-A', '--add_sns', action='store_true', help='Add series number suffix (compatible with -a)') # Parse arguments args = parser.parse_args() @@ -807,23 +594,19 @@ def main(): stb = DownloadShanoirDatasetToBIDS() stb.set_shanoir_username(args.username) stb.set_shanoir_domaine(args.domain) - stb.set_json_config_file( - json_file=args.config_file - ) # path to json configuration file + stb.set_json_config_file(json_file=args.config_file) # path to json configuration file stb.set_shanoir_file_type(shanoir_file_type=args.format) # Format (dicom or nifti) - stb.set_download_directory( - dl_dir=args.output_folder - ) # output folder (if None a default directory is created) + stb.set_download_directory(dl_dir=args.output_folder) # output folder (if None a default directory is created) if args.longitudinal: stb.toggle_longitudinal_version() if args.automri: stb.switch_to_automri_format() if args.add_sns: if not args.automri: - print("Warning : -A option is only compatible with -a option.") + print('Warning : -A option is only compatible with -a option.') stb.add_series_number_suffix() stb.download() -if __name__ == "__main__": +if __name__ == '__main__': main() From a7ac1eb8d0c75512dfa46e1215aa7e877745f566 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 26 Mar 2024 14:26:58 +0100 Subject: [PATCH 33/86] fix underscores in filename suffixes --- s2b_example_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 3b568a0..474faf7 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -5,7 +5,7 @@ [ {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", "bidsName": "acq-mprage_T1w"}, {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "acq-hr_T2w"}, - {"datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", "bidsName": "_task-restingState_acq-hipp_dir-AP_bold"}, + {"datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", "bidsName": "task-restingState_acq-hipp_dir-AP_bold"}, {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b3000_dir-AP_dwi"}, {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-AP_dwi"} ], From 29326f6cc36098f8f2c4730b40dba7d40607e230 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Thu, 28 Mar 2024 15:42:47 +0100 Subject: [PATCH 34/86] + --- s2b_example_config.json | 4 +-- shanoir2bids_heudiconv.py | 56 ++++++++++++++++++++++----------------- 2 files changed, 34 insertions(+), 26 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 474faf7..264f23f 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -9,8 +9,8 @@ {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b3000_dir-AP_dwi"}, {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-AP_dwi"} ], - "dcm2niix":"/home/qduche/Software/dcm2niix_lnx/dcm2niix", - "dcm2niix_options": "-v 0 -z y", + "dcm2niix":"~/softs/miniconda3/bin/dcm2niix", + "dcm2niix_options": {"verbose": false,"compress": "y", "anon_bids": true}, "find_and_replace_subject": [ {"find":"VS_Aneravimm_", "replace": "VS"}, diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index dcb5b55..f0d224d 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -24,6 +24,7 @@ import shanoir_downloader from dotenv import load_dotenv from heudiconv.main import workflow + # import loggger used in heudiconv workflow from heudiconv.main import lgr @@ -172,7 +173,7 @@ def read_json_config_file(json_file): def generate_heuristic_file( - shanoir2bids_dict: object, path_heuristic_file: object + shanoir2bids_dict: object, path_heuristic_file: object, output_type ) -> None: """Generate heudiconv heuristic.py file from shanoir2bids mapping dict Parameters @@ -180,11 +181,18 @@ def generate_heuristic_file( shanoir2bids_dict : path_heuristic_file : path of the python heuristic file (.py) """ + if output_type == 'dicom': + outtype = '("dicom",)' + elif output_type == 'nifti': + outtype = '("nii.gz",)' + else: + outtype = '("dicom","nii.gz")' + heuristic = f"""from heudiconv.heuristics.reproin import create_key def create_bids_key(dataset): - template = create_key(subdir=dataset['bidsDir'],file_suffix=r"run-{{item:02d}}_" + dataset['bidsName'],outtype=("dicom","nii.gz")) + template = create_key(subdir=dataset['bidsDir'],file_suffix=r"run-{{item:02d}}_" + dataset['bidsName'],outtype={outtype}) return template def get_dataset_to_key_mapping(shanoir2bids): @@ -248,6 +256,7 @@ def __init__(self): self.shanoir_file_type = ( DEFAULT_SHANOIR_FILE_TYPE # Default download type (nifti/dicom) ) + self.output_file_type = DEFAULT_SHANOIR_FILE_TYPE self.json_config_file = None self.list_fars = [] # List of substrings to edit in subjects names self.dl_dir = None # download directory, where data will be stored @@ -298,6 +307,12 @@ def set_shanoir_file_type(self, shanoir_file_type): else: sys.exit("Unknown shanoir file type {}".format(shanoir_file_type)) + def set_output_file_type(self, output_file_type): + if output_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, 'both']: + self.shanoir_file_type = output_file_type + else: + sys.exit("Unknown shanoir file type {}".format(output_file_type)) + def set_shanoir_study_id(self, study_id): self.shanoir_study_id = study_id @@ -354,18 +369,6 @@ def set_download_directory(self, dl_dir): Path(self.dl_dir).mkdir(parents=True, exist_ok=True) self.set_log_filename() - def set_heuristic_file(self, path_heuristic_file): - if path_heuristic_file is None: - print(f"No heuristic file provided") - else: - filename, ext = ops(path_heuristic_file) - if ext != ".py": - print( - f"Provided heuristic file {path_heuristic_file} is not a .py file as expected" - ) - else: - self.heuristic_file = path_heuristic_file - def set_log_filename(self): curr_time = datetime.datetime.now() basename = "shanoir_downloader_{:04}{:02}{:02}_{:02}{:02}{:02}.log".format( @@ -582,7 +585,7 @@ def download_subject(self, subject_to_search): mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" ) as heuristic_file: # Generate Heudiconv heuristic file from configuration.json mapping - generate_heuristic_file(bids_mapping, heuristic_file.name) + generate_heuristic_file(bids_mapping, heuristic_file.name, output_type=self.output_file_type) with tempfile.NamedTemporaryFile( mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" ) as dcm2niix_config_file: @@ -641,13 +644,20 @@ def main(): default="shanoir.irisa.fr", help="The shanoir domain to query.", ) + # parser.add_argument( + # "-f", + # "--format", + # default="dicom", + # choices=["dicom"], + # help="The format to download.", + # ) parser.add_argument( - "-f", - "--format", - default="dicom", - choices=["dicom"], + "--outformat", + default="nifti", + choices=["nifti", "dicom", "both"], help="The format to download.", ) + shanoir_downloader.add_output_folder_argument(parser=parser, required=False) # Add the argument for the configuration file parser.add_argument( @@ -663,6 +673,7 @@ def main(): action="store_true", help="Toggle longitudinal approach.", ) + # parser.add_argument( # "-a", "--automri", action="store_true", help="Switch to automri file tree." # ) @@ -682,14 +693,11 @@ def main(): stb.set_json_config_file( json_file=args.config_file ) # path to json configuration file - stb.set_shanoir_file_type(shanoir_file_type=args.format) # Format (dicom or nifti) + stb.set_output_file_type(output_file_type=args.outformat) stb.set_download_directory( dl_dir=args.output_folder ) # output folder (if None a default directory is created) - # stb.set_heuristic_file(path_heuristic_file="/home/alpron/heuristic.py") - # stb.set_dcm2niix_config_files( - # path_dcm2niix_options_files="/home/alpron/dcm2niix_options.json" - # ) + if args.longitudinal: stb.toggle_longitudinal_version() # if args.automri: From 58e17d27d379e9b0cd6df1630f833397941a1beb Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 8 Apr 2024 16:21:05 +0200 Subject: [PATCH 35/86] clean parser options --- shanoir2bids_heudiconv.py | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index f0d224d..1d58184 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -384,12 +384,6 @@ def set_log_filename(self): def toggle_longitudinal_version(self): self.longitudinal = True - def switch_to_automri_format(self): - self.to_automri_format = True - - def add_series_number_suffix(self): - self.add_sns = True - def configure_parser(self): """ Configure the parser and the configuration of the shanoir_downloader @@ -418,7 +412,6 @@ def download_subject(self, subject_to_search): # temporary directory containing dowloaded DICOM.zip files with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_dicom: with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_archive: - print(tmp_archive) # Loop on each sequence defined in the dictionary for seq in range(self.n_seq): # Isolate elements that are called many times @@ -644,13 +637,7 @@ def main(): default="shanoir.irisa.fr", help="The shanoir domain to query.", ) - # parser.add_argument( - # "-f", - # "--format", - # default="dicom", - # choices=["dicom"], - # help="The format to download.", - # ) + parser.add_argument( "--outformat", default="nifti", @@ -674,16 +661,6 @@ def main(): help="Toggle longitudinal approach.", ) - # parser.add_argument( - # "-a", "--automri", action="store_true", help="Switch to automri file tree." - # ) - # parser.add_argument( - # "-A", - # "--add_sns", - # action="store_true", - # help="Add series number suffix (compatible with -a)", - # ) - # Parse arguments args = parser.parse_args() # Start configuring the DownloadShanoirDatasetToBids class instance @@ -700,12 +677,6 @@ def main(): if args.longitudinal: stb.toggle_longitudinal_version() - # if args.automri: - # stb.switch_to_automri_format() - # if args.add_sns: - # if not args.automri: - # print("Warning : -A option is only compatible with -a option.") - # stb.add_series_number_suffix() stb.download() From 86fced9e46fa2206c9a0cdb3027d1e4f05ebb508 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 9 Apr 2024 10:53:14 +0200 Subject: [PATCH 36/86] clean parser options --- s2b_example_config.json | 2 +- shanoir2bids_heudiconv.py | 35 ++++++++++++++--------------------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 264f23f..86e4041 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -9,7 +9,7 @@ {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b3000_dir-AP_dwi"}, {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-AP_dwi"} ], - "dcm2niix":"~/softs/miniconda3/bin/dcm2niix", + "dcm2niix":"/home/alpron/softs/miniconda3/bin/dcm2niix", "dcm2niix_options": {"verbose": false,"compress": "y", "anon_bids": true}, "find_and_replace_subject": [ diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index 1d58184..fe363ff 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -173,7 +173,7 @@ def read_json_config_file(json_file): def generate_heuristic_file( - shanoir2bids_dict: object, path_heuristic_file: object, output_type + shanoir2bids_dict: object, path_heuristic_file: object, output_type='("dicom","nii.gz")' ) -> None: """Generate heudiconv heuristic.py file from shanoir2bids mapping dict Parameters @@ -233,7 +233,6 @@ def infotodict(seqinfo): with open(path_heuristic_file, "w", encoding="utf-8") as file: file.write(heuristic) - file.close() pass @@ -253,10 +252,8 @@ def __init__(self): self.shanoir_username = None # Shanoir username self.shanoir_study_id = None # Shanoir study ID self.shanoir_session_id = None # Shanoir study ID - self.shanoir_file_type = ( - DEFAULT_SHANOIR_FILE_TYPE # Default download type (nifti/dicom) - ) - self.output_file_type = DEFAULT_SHANOIR_FILE_TYPE + self.shanoir_file_type = SHANOIR_FILE_TYPE_DICOM # Download File Type (DICOM) + self.output_file_type = DEFAULT_SHANOIR_FILE_TYPE # Default Export File Type (NIFTI) self.json_config_file = None self.list_fars = [] # List of substrings to edit in subjects names self.dl_dir = None # download directory, where data will be stored @@ -301,17 +298,11 @@ def set_json_config_file(self, json_file): self.set_date_from(date_from=date_from) self.set_date_to(date_to=date_to) - def set_shanoir_file_type(self, shanoir_file_type): - if shanoir_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI]: - self.shanoir_file_type = shanoir_file_type - else: - sys.exit("Unknown shanoir file type {}".format(shanoir_file_type)) - - def set_output_file_type(self, output_file_type): - if output_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, 'both']: - self.shanoir_file_type = output_file_type + def set_output_file_type(self, outfile_type): + if outfile_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, 'both']: + self.output_file_type = outfile_type else: - sys.exit("Unknown shanoir file type {}".format(output_file_type)) + sys.exit("Unknown output file type {}".format(outfile_type)) def set_shanoir_study_id(self, study_id): self.shanoir_study_id = study_id @@ -389,7 +380,11 @@ def configure_parser(self): Configure the parser and the configuration of the shanoir_downloader """ self.parser = shanoir_downloader.create_arg_parser() - shanoir_downloader.add_common_arguments(self.parser) + shanoir_downloader.add_username_argument(self.parser) + shanoir_downloader.add_domain_argument(self.parser) + self.parser.add_argument('-f', '--format', default='dicom', choices=['dicom'], + help='The format to download.') + shanoir_downloader.add_output_folder_argument(self.parser) shanoir_downloader.add_configuration_arguments(self.parser) shanoir_downloader.add_search_arguments(self.parser) shanoir_downloader.add_ids_arguments(self.parser) @@ -601,8 +596,6 @@ def download_subject(self, subject_to_search): workflow_params["session"] = bids_seq_session workflow(**workflow_params) - # TODO add nipype logging into shanoir log file ? - # TODO use provenance option ? currently not working properly fp.close() def download(self): @@ -640,7 +633,7 @@ def main(): parser.add_argument( "--outformat", - default="nifti", + default="both", choices=["nifti", "dicom", "both"], help="The format to download.", ) @@ -670,7 +663,7 @@ def main(): stb.set_json_config_file( json_file=args.config_file ) # path to json configuration file - stb.set_output_file_type(output_file_type=args.outformat) + stb.set_output_file_type(args.outformat) stb.set_download_directory( dl_dir=args.output_folder ) # output folder (if None a default directory is created) From f1a1878ff4e425752d312e6d317cb08db34c5ea8 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 15 Apr 2024 14:11:01 +0200 Subject: [PATCH 37/86] added dcm2niix executable check --- shanoir2bids_heudiconv.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index fe363ff..fdd0820 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -261,6 +261,7 @@ def __init__(self): self.n_seq = 0 # Number of sequences in the shanoir2bids_dict self.log_fn = None self.dcm2niix_path = None # Path to the dcm2niix the user wants to use + self.actual_dcm2niix_path = shutil.which('dcm2niix') self.dcm2niix_opts = None # Options to add to the dcm2niix call self.date_from = None self.date_to = None @@ -375,6 +376,14 @@ def set_log_filename(self): def toggle_longitudinal_version(self): self.longitudinal = True + def is_correct_dcm2niix(self): + current_version = Path(self.actual_dcm2niix_path) + config_version = Path(self.dcm2niix_path) + if current_version is not None and config_version is not None: + return config_version.samefile(current_version) + else: + return False + def configure_parser(self): """ Configure the parser and the configuration of the shanoir_downloader @@ -670,8 +679,10 @@ def main(): if args.longitudinal: stb.toggle_longitudinal_version() - - stb.download() + if not stb.is_correct_dcm2niix(): + print(f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}") + else: + stb.download() if __name__ == "__main__": From 0cf37ddbd6166985f1ab70cbcddbf9047c6df5bd Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 22 Apr 2024 12:18:23 +0200 Subject: [PATCH 38/86] added back automri support option --- shanoir2bids_heudiconv.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index fdd0820..a8ec3e2 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -373,6 +373,9 @@ def set_log_filename(self): ) self.log_fn = opj(self.dl_dir, basename) + def switch_to_automri_format(self): + self.to_automri_format = True + def toggle_longitudinal_version(self): self.longitudinal = True @@ -603,6 +606,8 @@ def download_subject(self, subject_to_search): if self.longitudinal: workflow_params["session"] = bids_seq_session + if self.to_automri_format: + workflow_params["bids_options"] = None workflow(**workflow_params) fp.close() @@ -663,6 +668,8 @@ def main(): help="Toggle longitudinal approach.", ) + parser.add_argument('-a', '--automri', action='store_true', help='Switch to automri file tree.') + args = parser.parse_args() # Start configuring the DownloadShanoirDatasetToBids class instance @@ -679,6 +686,8 @@ def main(): if args.longitudinal: stb.toggle_longitudinal_version() + if args.automri: + stb.switch_to_automri_format() if not stb.is_correct_dcm2niix(): print(f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}") else: From ed2b86be0847d2b65024bd9b5b9b26700cfcf474 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 22 Apr 2024 12:21:32 +0200 Subject: [PATCH 39/86] Revert "Merge branch 'bids-1.9.0-compliance' into heudiconv" This reverts commit 2c407217a0ea105f071f7f6c45a10458f28207b9, reversing changes made to 7f78df14532e45280b6c46323fd268c6fe9b1ca6. remove modifications made to configuration files to make them BIDS compliant --- s2b_example_config.json | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 86e4041..617ae20 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -3,11 +3,11 @@ "subjects": ["VS_Aneravimm_010", "VS_Aneravimm_011"], "data_to_bids": [ - {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", "bidsName": "acq-mprage_T1w"}, - {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "acq-hr_T2w"}, - {"datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", "bidsName": "task-restingState_acq-hipp_dir-AP_bold"}, - {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b3000_dir-AP_dwi"}, - {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-AP_dwi"} + {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", "bidsName": "t1w-mprage"}, + {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "t2-hr"}, + {"datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", "bidsName": "ap-hipp"}, + {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "cusp66-ap-b3000"}, + {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "cusp66-ap-b0"} ], "dcm2niix":"/home/alpron/softs/miniconda3/bin/dcm2niix", "dcm2niix_options": {"verbose": false,"compress": "y", "anon_bids": true}, From 1bd43df052eef4e4aac454305a252ce0ef44171e Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 22 Apr 2024 12:46:40 +0200 Subject: [PATCH 40/86] added main dcm2niix configuration options into nipype format + documentation link as comment entry --- s2b_example_config.json | 61 ++++++++++++++++++++--------- s2b_example_config_EMISEP_long.json | 13 +++++- 2 files changed, 55 insertions(+), 19 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 617ae20..0746b5e 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -1,19 +1,44 @@ { - "study_name": "Aneravimm", - "subjects": ["VS_Aneravimm_010", "VS_Aneravimm_011"], - "data_to_bids": - [ - {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", "bidsName": "t1w-mprage"}, - {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "t2-hr"}, - {"datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", "bidsName": "ap-hipp"}, - {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "cusp66-ap-b3000"}, - {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "cusp66-ap-b0"} - ], - "dcm2niix":"/home/alpron/softs/miniconda3/bin/dcm2niix", - "dcm2niix_options": {"verbose": false,"compress": "y", "anon_bids": true}, - "find_and_replace_subject": - [ - {"find":"VS_Aneravimm_", "replace": "VS"}, - {"find":"Vs_Aneravimm_", "replace": "VS"} - ] -} \ No newline at end of file + "study_name": "Aneravimm", + "subjects": ["VS_Aneravimm_010", "VS_Aneravimm_011"], + "data_to_bids": [ + { + "datasetName": "t1_mprage_sag_p2_iso", + "bidsDir": "anat", + "bidsName": "t1w-mprage", + }, + {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "t2-hr"}, + { + "datasetName": "Resting State_bold AP 1.6mm HIPP", + "bidsDir": "func", + "bidsName": "ap-hipp" + }, + { + "datasetName": "Diff cusp66 b3000 AP 1.5mm", + "bidsDir": "dwi", + "bidsName": "cusp66-ap-b3000" + }, + { + "datasetName": "Diff cusp66 b0 PA 1.5mm", + "bidsDir": "dwi", + "bidsName": "cusp66-ap-b0" + } + ], + "dcm2niix": "/home/alpron/softs/miniconda3/bin/dcm2niix", + "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", + "dcm2niix_options": { + "bids_format": true, + "anon_bids": true, + "compress": "y", + "compression": 5, + "crop": false, + "has_private": false, + "ignore_deriv": false, + "single_file": false, + "verbose": false + }, + "find_and_replace_subject": [ + {"find": "VS_Aneravimm_", "replace": "VS"}, + {"find": "Vs_Aneravimm_", "replace": "VS"} + ] +} diff --git a/s2b_example_config_EMISEP_long.json b/s2b_example_config_EMISEP_long.json index bc64ab7..c7bfb4a 100644 --- a/s2b_example_config_EMISEP_long.json +++ b/s2b_example_config_EMISEP_long.json @@ -64,7 +64,18 @@ ], "dcm2niix":"~/softs/miniconda3/bin/dcm2niix", - "dcm2niix_options": {"verbose": true,"compress": "y", "anon_bids": true} + "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", + "dcm2niix_options": { + "bids_format": true, + "anon_bids": true, + "compress": "y", + "compression": 5, + "crop": false, + "has_private": false, + "ignore_deriv": false, + "single_file": false, + "verbose": true + } } From 886eb1dad841f2e1eea3cc7c52d899c3bcca6384 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 22 Apr 2024 12:47:48 +0200 Subject: [PATCH 41/86] fix authors --- shanoir2bids_heudiconv.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index a8ec3e2..645b45e 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -4,8 +4,7 @@ The script is made to run for every project given some information provided by the user into a ".json" configuration file. More details regarding the configuration file in the Readme.md""" # Script to download and BIDS-like organize data on Shanoir using "shanoir_downloader.py" developed by Arthur Masson -# @Author: Malo Gaubert , Quentin Duché -# @Date: 24 Juin 2022 + import os from os.path import join as opj, splitext as ops, exists as ope, dirname as opd From 32027154174e6d238d1279283d3314dd543682e6 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 21 May 2024 10:39:00 +0200 Subject: [PATCH 42/86] [FIX]: typo --- s2b_example_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 0746b5e..e03f55d 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -5,7 +5,7 @@ { "datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", - "bidsName": "t1w-mprage", + "bidsName": "t1w-mprage" }, {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "t2-hr"}, { From 4b4dc06b9cbe6c7a6560d6b76a5ea247f2b3a8a7 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 21 May 2024 14:18:46 +0200 Subject: [PATCH 43/86] [ENH]: not working hack for heudiconv support --- shanoir2bids_heudiconv.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index 645b45e..8a20817 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -609,6 +609,19 @@ def download_subject(self, subject_to_search): workflow_params["bids_options"] = None workflow(**workflow_params) + if self.to_automri_format: + # horrible hack to adapt to automri ontology + dicoms = glob(opj(self.dl_dir, str(self.shanoir_study_id), "**", "*.dcm"), recursive=True) + niftis = glob(opj(self.dl_dir, str(self.shanoir_study_id), "**", "*.nii.gz"), recursive=True) + export_files = dicoms + niftis + to_modify_files = [f for f in export_files if not '.git' in f] + for f in to_modify_files: + new_file = f.replace('/' + subject_id + '/', '/' ) + new_file = new_file.replace('sub-','su_') + os.system('git mv ' + f + ' ' + new_file) + from datalad.api import save + save(path=opj(self.dl_dir, str(self.shanoir_study_id)), recursive=True, message='reformat into automri standart') + fp.close() def download(self): From b13bd8e4a1b82feae17d2f4689dedec88790517d Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 21 May 2024 14:22:54 +0200 Subject: [PATCH 44/86] [LINT]: black linting these files --- s2b_example_config.json | 16 +- s2b_example_config_EMISEP_long.json | 367 ++++++++++++++++++++++------ shanoir2bids_heudiconv.py | 67 +++-- 3 files changed, 354 insertions(+), 96 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index e03f55d..4e868fd 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -5,24 +5,24 @@ { "datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", - "bidsName": "t1w-mprage" + "bidsName": "t1w-mprage", }, {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "t2-hr"}, { "datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", - "bidsName": "ap-hipp" + "bidsName": "ap-hipp", }, { "datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", - "bidsName": "cusp66-ap-b3000" + "bidsName": "cusp66-ap-b3000", }, { "datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", - "bidsName": "cusp66-ap-b0" - } + "bidsName": "cusp66-ap-b0", + }, ], "dcm2niix": "/home/alpron/softs/miniconda3/bin/dcm2niix", "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", @@ -35,10 +35,10 @@ "has_private": false, "ignore_deriv": false, "single_file": false, - "verbose": false + "verbose": false, }, "find_and_replace_subject": [ {"find": "VS_Aneravimm_", "replace": "VS"}, - {"find": "Vs_Aneravimm_", "replace": "VS"} - ] + {"find": "Vs_Aneravimm_", "replace": "VS"}, + ], } diff --git a/s2b_example_config_EMISEP_long.json b/s2b_example_config_EMISEP_long.json index c7bfb4a..d000628 100644 --- a/s2b_example_config_EMISEP_long.json +++ b/s2b_example_config_EMISEP_long.json @@ -1,70 +1,299 @@ { - "study_name": "EMISEP", - "subjects": ["08-74"], - "session": "08-74 MO ENCEPHALE", - "data_to_bids": - [ - {"datasetName": "3D T1 MPRAGE", "bidsDir": "brain", "bidsName": "T1w", "bidsSession": "M00"}, - {"datasetName": "*3d flair*", "bidsDir": "brain", "bidsName": "FLAIR", "bidsSession": "M00"}, - {"datasetName": "*c1c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", "bidsSession": "M00"}, - {"datasetName": "*c1-c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", "bidsSession": "M00"}, - - {"datasetName": "*c4c7*", "bidsDir": "spinalCord", "bidsName": "t2ax_C4C7", "bidsSession": "M00"}, - {"datasetName": "*c4-c7*", "bidsDir": "spinalCord", "bidsName": "t2ax_C4C7", "bidsSession": "M00"}, - - - {"datasetName": "t2_tse_rr_p2_sag", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "t2_tse_rr_p2_sag2.5 te 81", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_384", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_p2", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "SAG T2 CERV", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "SAG T2 DORSAL", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - - {"datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "t2_tse_rr_p2_sag_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_384_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260_comp_ad", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_p2_comp_ad", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "SAG T2 TSE MOELLE", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - - - - {"datasetName": "t1_fl3d_sag_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_sag_p2_iso mt", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_tra_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_tra_p2_iso mt", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso mt tr38", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso mt tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso mt_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso tr38", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat_mt", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38_sans_sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_m0_iso tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_mt_iso tr38", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - - - {"datasetName": "3d_t1_mprage_sag", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso cerv", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso cerv_rr", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso ofsep", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_axial", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_cor", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_tra", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso ofsep_rr", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "T1 MPRAGE CERV", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"} - - ], - "dcm2niix":"~/softs/miniconda3/bin/dcm2niix", - "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", + "study_name": "EMISEP", + "subjects": ["08-74"], + "session": "08-74 MO ENCEPHALE", + "data_to_bids": [ + { + "datasetName": "3D T1 MPRAGE", + "bidsDir": "brain", + "bidsName": "T1w", + "bidsSession": "M00", + }, + { + "datasetName": "*3d flair*", + "bidsDir": "brain", + "bidsName": "FLAIR", + "bidsSession": "M00", + }, + { + "datasetName": "*c1c3*", + "bidsDir": "spinalCord", + "bidsName": "t2ax_C1C3", + "bidsSession": "M00", + }, + { + "datasetName": "*c1-c3*", + "bidsDir": "spinalCord", + "bidsName": "t2ax_C1C3", + "bidsSession": "M00", + }, + { + "datasetName": "*c4c7*", + "bidsDir": "spinalCord", + "bidsName": "t2ax_C4C7", + "bidsSession": "M00", + }, + { + "datasetName": "*c4-c7*", + "bidsDir": "spinalCord", + "bidsName": "t2ax_C4C7", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_rr_p2_sag", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_rr_p2_sag2.5 te 81", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_384", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_p2", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "SAG T2 CERV", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "SAG T2 DORSAL", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp_sp", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_rr_p2_sag_comp_sp", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_384_comp_sp", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260_comp_ad", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_p2_comp_ad", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "SAG T2 TSE MOELLE", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_sag_p2_iso", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_sag_p2_iso mt", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_tra_p2_iso", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_tra_p2_iso mt", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso mt tr38", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso mt tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso mt_tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso tr38", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat_mt", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38_sans_sat", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso_tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_m0_iso tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_mt_iso tr38", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "3d_t1_mprage_sag", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso cerv", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso cerv_rr", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso ofsep", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_axial", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_cor", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_tra", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso ofsep_rr", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "T1 MPRAGE CERV", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + ], + "dcm2niix": "~/softs/miniconda3/bin/dcm2niix", + "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", "dcm2niix_options": { "bids_format": true, "anon_bids": true, @@ -74,8 +303,6 @@ "has_private": false, "ignore_deriv": false, "single_file": false, - "verbose": true - } + "verbose": true, + }, } - - diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index 8a20817..19c7427 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -172,7 +172,9 @@ def read_json_config_file(json_file): def generate_heuristic_file( - shanoir2bids_dict: object, path_heuristic_file: object, output_type='("dicom","nii.gz")' + shanoir2bids_dict: object, + path_heuristic_file: object, + output_type='("dicom","nii.gz")', ) -> None: """Generate heudiconv heuristic.py file from shanoir2bids mapping dict Parameters @@ -180,9 +182,9 @@ def generate_heuristic_file( shanoir2bids_dict : path_heuristic_file : path of the python heuristic file (.py) """ - if output_type == 'dicom': + if output_type == "dicom": outtype = '("dicom",)' - elif output_type == 'nifti': + elif output_type == "nifti": outtype = '("nii.gz",)' else: outtype = '("dicom","nii.gz")' @@ -252,7 +254,9 @@ def __init__(self): self.shanoir_study_id = None # Shanoir study ID self.shanoir_session_id = None # Shanoir study ID self.shanoir_file_type = SHANOIR_FILE_TYPE_DICOM # Download File Type (DICOM) - self.output_file_type = DEFAULT_SHANOIR_FILE_TYPE # Default Export File Type (NIFTI) + self.output_file_type = ( + DEFAULT_SHANOIR_FILE_TYPE # Default Export File Type (NIFTI) + ) self.json_config_file = None self.list_fars = [] # List of substrings to edit in subjects names self.dl_dir = None # download directory, where data will be stored @@ -260,7 +264,7 @@ def __init__(self): self.n_seq = 0 # Number of sequences in the shanoir2bids_dict self.log_fn = None self.dcm2niix_path = None # Path to the dcm2niix the user wants to use - self.actual_dcm2niix_path = shutil.which('dcm2niix') + self.actual_dcm2niix_path = shutil.which("dcm2niix") self.dcm2niix_opts = None # Options to add to the dcm2niix call self.date_from = None self.date_to = None @@ -299,7 +303,7 @@ def set_json_config_file(self, json_file): self.set_date_to(date_to=date_to) def set_output_file_type(self, outfile_type): - if outfile_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, 'both']: + if outfile_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, "both"]: self.output_file_type = outfile_type else: sys.exit("Unknown output file type {}".format(outfile_type)) @@ -393,8 +397,13 @@ def configure_parser(self): self.parser = shanoir_downloader.create_arg_parser() shanoir_downloader.add_username_argument(self.parser) shanoir_downloader.add_domain_argument(self.parser) - self.parser.add_argument('-f', '--format', default='dicom', choices=['dicom'], - help='The format to download.') + self.parser.add_argument( + "-f", + "--format", + default="dicom", + choices=["dicom"], + help="The format to download.", + ) shanoir_downloader.add_output_folder_argument(self.parser) shanoir_downloader.add_configuration_arguments(self.parser) shanoir_downloader.add_search_arguments(self.parser) @@ -584,7 +593,9 @@ def download_subject(self, subject_to_search): mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" ) as heuristic_file: # Generate Heudiconv heuristic file from configuration.json mapping - generate_heuristic_file(bids_mapping, heuristic_file.name, output_type=self.output_file_type) + generate_heuristic_file( + bids_mapping, heuristic_file.name, output_type=self.output_file_type + ) with tempfile.NamedTemporaryFile( mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" ) as dcm2niix_config_file: @@ -611,16 +622,32 @@ def download_subject(self, subject_to_search): workflow(**workflow_params) if self.to_automri_format: # horrible hack to adapt to automri ontology - dicoms = glob(opj(self.dl_dir, str(self.shanoir_study_id), "**", "*.dcm"), recursive=True) - niftis = glob(opj(self.dl_dir, str(self.shanoir_study_id), "**", "*.nii.gz"), recursive=True) + dicoms = glob( + opj(self.dl_dir, str(self.shanoir_study_id), "**", "*.dcm"), + recursive=True, + ) + niftis = glob( + opj( + self.dl_dir, + str(self.shanoir_study_id), + "**", + "*.nii.gz", + ), + recursive=True, + ) export_files = dicoms + niftis - to_modify_files = [f for f in export_files if not '.git' in f] + to_modify_files = [f for f in export_files if not ".git" in f] for f in to_modify_files: - new_file = f.replace('/' + subject_id + '/', '/' ) - new_file = new_file.replace('sub-','su_') - os.system('git mv ' + f + ' ' + new_file) + new_file = f.replace("/" + subject_id + "/", "/") + new_file = new_file.replace("sub-", "su_") + os.system("git mv " + f + " " + new_file) from datalad.api import save - save(path=opj(self.dl_dir, str(self.shanoir_study_id)), recursive=True, message='reformat into automri standart') + + save( + path=opj(self.dl_dir, str(self.shanoir_study_id)), + recursive=True, + message="reformat into automri standart", + ) fp.close() @@ -680,7 +707,9 @@ def main(): help="Toggle longitudinal approach.", ) - parser.add_argument('-a', '--automri', action='store_true', help='Switch to automri file tree.') + parser.add_argument( + "-a", "--automri", action="store_true", help="Switch to automri file tree." + ) args = parser.parse_args() @@ -701,7 +730,9 @@ def main(): if args.automri: stb.switch_to_automri_format() if not stb.is_correct_dcm2niix(): - print(f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}") + print( + f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}" + ) else: stb.download() From 0c7c351a9633b8f25dff870a2c00b4d89822b078 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Wed, 22 May 2024 14:36:38 +0200 Subject: [PATCH 45/86] [FIX] modified heuristic file to support automri --- shanoir2bids_heudiconv.py | 121 +++++++++++++++++++++++++++----------- 1 file changed, 87 insertions(+), 34 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index 19c7427..6762a73 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -170,10 +170,88 @@ def read_json_config_file(json_file): date_to, ) +def generate_automri_heuristic_file( + shanoir2bids_dict, + path_heuristic_file, + subject, + output_type='("dicom","nii.gz")', + session=None): + if output_type == "dicom": + outtype = '("dicom",)' + elif output_type == "nifti": + outtype = '("nii.gz",)' + else: + outtype = '("dicom","nii.gz")' + + heuristic = f""" + def create_key( + subdir: Optional[str], + file_suffix: str, + outtype: tuple[str, ...] = ("nii.gz", "dicom"), + annotation_classes: None = None, + prefix: str = "", +) -> tuple[str, tuple[str, ...], None]: + if not subdir: + raise ValueError("subdir must be a valid format string") + template = os.path.join( + prefix, + "{subject}", + subdir, + "su_{subject}_%s" % file_suffix, + ) + return template, outtype, annotation_classes + + def create_bids_key(dataset): + + template = create_key(subdir=dataset['bidsDir'],file_suffix=r"run-{{item:02d}}_" + dataset['bidsName'],outtype={outtype}) + return template + + def get_dataset_to_key_mapping(shanoir2bids): + dataset_to_key = dict() + for dataset in shanoir2bids: + template = create_bids_key(dataset) + dataset_to_key[dataset['datasetName']] = template + return dataset_to_key + + def simplify_runs(info): + info_final = dict() + for key in info.keys(): + if len(info[key])==1: + new_template = key[0].replace('run-{{item:02d}}_','') + new_key = (new_template, key[1], key[2]) + info_final[new_key] = info[key] + else: + info_final[key] = info[key] + return info_final + + def infotodict(seqinfo): + + info = dict() + shanoir2bids = {shanoir2bids_dict} + + dataset_to_key = get_dataset_to_key_mapping(shanoir2bids) + for seq in seqinfo: + if seq.series_description in dataset_to_key.keys(): + key = dataset_to_key[seq.series_description] + if key in info.keys(): + info[key].append(seq.series_id) + else: + info[key] = [seq.series_id] + # remove run- key if not needed (one run only) + info_final = simplify_runs(info) + return info_final + """ + + with open(path_heuristic_file, "w", encoding="utf-8") as file: + file.write(heuristic) + pass + -def generate_heuristic_file( - shanoir2bids_dict: object, - path_heuristic_file: object, + +def generate_bids_heuristic_file( + shanoir2bids_dict, + + path_heuristic_file, output_type='("dicom","nii.gz")', ) -> None: """Generate heudiconv heuristic.py file from shanoir2bids mapping dict @@ -517,9 +595,11 @@ def download_subject(self, subject_to_search): # If the user has defined a list of edits to subject names... then do the find and replace for far in self.list_fars: su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) - # ID of the subject (sub-*) subject_id = su_id + + if self.to_automri_format: + subject_id = 'su_' + su_id # correct BIDS mapping of the searched dataset bids_seq_mapping = { "datasetName": item["datasetName"], @@ -593,7 +673,9 @@ def download_subject(self, subject_to_search): mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" ) as heuristic_file: # Generate Heudiconv heuristic file from configuration.json mapping - generate_heuristic_file( + if self.to_automri_format: + generate_automri_heuristic_file(bids_mapping,heuristic_file.name, subject_id,self.output_file_type, bids_seq_session) + generate_bids_heuristic_file( bids_mapping, heuristic_file.name, output_type=self.output_file_type ) with tempfile.NamedTemporaryFile( @@ -620,35 +702,6 @@ def download_subject(self, subject_to_search): workflow_params["bids_options"] = None workflow(**workflow_params) - if self.to_automri_format: - # horrible hack to adapt to automri ontology - dicoms = glob( - opj(self.dl_dir, str(self.shanoir_study_id), "**", "*.dcm"), - recursive=True, - ) - niftis = glob( - opj( - self.dl_dir, - str(self.shanoir_study_id), - "**", - "*.nii.gz", - ), - recursive=True, - ) - export_files = dicoms + niftis - to_modify_files = [f for f in export_files if not ".git" in f] - for f in to_modify_files: - new_file = f.replace("/" + subject_id + "/", "/") - new_file = new_file.replace("sub-", "su_") - os.system("git mv " + f + " " + new_file) - from datalad.api import save - - save( - path=opj(self.dl_dir, str(self.shanoir_study_id)), - recursive=True, - message="reformat into automri standart", - ) - fp.close() def download(self): From 77f8e9e719bf5458c326fdd702d37cc41877dd2a Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Wed, 22 May 2024 14:58:17 +0200 Subject: [PATCH 46/86] [FIX] fixed typo introduced by black linting in json file --- s2b_example_config.json | 14 ++-- s2b_example_config_EMISEP_long.json | 102 ++++++++++++++-------------- 2 files changed, 58 insertions(+), 58 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 4e868fd..c8e3f13 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -5,23 +5,23 @@ { "datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", - "bidsName": "t1w-mprage", + "bidsName": "t1w-mprage" }, {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "t2-hr"}, { "datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", - "bidsName": "ap-hipp", + "bidsName": "ap-hipp" }, { "datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", - "bidsName": "cusp66-ap-b3000", + "bidsName": "cusp66-ap-b3000" }, { "datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", - "bidsName": "cusp66-ap-b0", + "bidsName": "cusp66-ap-b0" }, ], "dcm2niix": "/home/alpron/softs/miniconda3/bin/dcm2niix", @@ -35,10 +35,10 @@ "has_private": false, "ignore_deriv": false, "single_file": false, - "verbose": false, + "verbose": false }, "find_and_replace_subject": [ {"find": "VS_Aneravimm_", "replace": "VS"}, - {"find": "Vs_Aneravimm_", "replace": "VS"}, - ], + {"find": "Vs_Aneravimm_", "replace": "VS"} + ] } diff --git a/s2b_example_config_EMISEP_long.json b/s2b_example_config_EMISEP_long.json index d000628..e22a037 100644 --- a/s2b_example_config_EMISEP_long.json +++ b/s2b_example_config_EMISEP_long.json @@ -7,290 +7,290 @@ "datasetName": "3D T1 MPRAGE", "bidsDir": "brain", "bidsName": "T1w", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "*3d flair*", "bidsDir": "brain", "bidsName": "FLAIR", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "*c1c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "*c1-c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "*c4c7*", "bidsDir": "spinalCord", "bidsName": "t2ax_C4C7", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "*c4-c7*", "bidsDir": "spinalCord", "bidsName": "t2ax_C4C7", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_rr_p2_sag", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_rr_p2_sag2.5 te 81", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_384", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_p2", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "SAG T2 CERV", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "SAG T2 DORSAL", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_rr_p2_sag_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_384_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260_comp_ad", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_p2_comp_ad", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "SAG T2 TSE MOELLE", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_sag_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_sag_p2_iso mt", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_tra_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_tra_p2_iso mt", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso mt tr38", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso mt tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso mt_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso tr38", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat_mt", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38_sans_sat", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_m0_iso tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_mt_iso tr38", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "3d_t1_mprage_sag", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso cerv", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso cerv_rr", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso ofsep", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_axial", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_cor", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_tra", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso ofsep_rr", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "T1 MPRAGE CERV", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", - }, + "bidsSession": "M00" + } ], "dcm2niix": "~/softs/miniconda3/bin/dcm2niix", "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", @@ -303,6 +303,6 @@ "has_private": false, "ignore_deriv": false, "single_file": false, - "verbose": true, - }, + "verbose": true + } } From 9feb9249d991320bf321ee9c21d266dfe9660e5a Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Wed, 22 May 2024 16:13:06 +0200 Subject: [PATCH 47/86] [FIX] fixed typo introduced by black linting in json file --- s2b_example_config.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index c8e3f13..e03f55d 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -22,7 +22,7 @@ "datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "cusp66-ap-b0" - }, + } ], "dcm2niix": "/home/alpron/softs/miniconda3/bin/dcm2niix", "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", From 2672fc4e20defee0baf5ae88585ee84e1942ef0a Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Wed, 22 May 2024 16:14:11 +0200 Subject: [PATCH 48/86] [ENH]: v2 of automri generation. Not to be used (back up solution) use post processing parsing --- shanoir2bids_heudiconv.py | 129 +++++++++++++++++++++----------------- 1 file changed, 70 insertions(+), 59 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index 6762a73..c7c561c 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -170,12 +170,14 @@ def read_json_config_file(json_file): date_to, ) + def generate_automri_heuristic_file( - shanoir2bids_dict, - path_heuristic_file, - subject, - output_type='("dicom","nii.gz")', - session=None): + shanoir2bids_dict, + path_heuristic_file, + subject, + output_type='("dicom","nii.gz")', + session=None, +): if output_type == "dicom": outtype = '("dicom",)' elif output_type == "nifti": @@ -183,63 +185,63 @@ def generate_automri_heuristic_file( else: outtype = '("dicom","nii.gz")' - heuristic = f""" - def create_key( - subdir: Optional[str], - file_suffix: str, - outtype: tuple[str, ...] = ("nii.gz", "dicom"), - annotation_classes: None = None, - prefix: str = "", -) -> tuple[str, tuple[str, ...], None]: + heuristic = f"""import os + +def create_key( + subdir, + file_suffix, + outtype= ("nii.gz", "dicom"), + annotation_classes= None, + prefix= "", +): if not subdir: raise ValueError("subdir must be a valid format string") template = os.path.join( prefix, - "{subject}", subdir, - "su_{subject}_%s" % file_suffix, + "{subject}_%s" % file_suffix, ) return template, outtype, annotation_classes - def create_bids_key(dataset): - - template = create_key(subdir=dataset['bidsDir'],file_suffix=r"run-{{item:02d}}_" + dataset['bidsName'],outtype={outtype}) - return template - - def get_dataset_to_key_mapping(shanoir2bids): - dataset_to_key = dict() - for dataset in shanoir2bids: - template = create_bids_key(dataset) - dataset_to_key[dataset['datasetName']] = template - return dataset_to_key - - def simplify_runs(info): - info_final = dict() - for key in info.keys(): - if len(info[key])==1: - new_template = key[0].replace('run-{{item:02d}}_','') - new_key = (new_template, key[1], key[2]) - info_final[new_key] = info[key] - else: - info_final[key] = info[key] - return info_final +def create_bids_key(dataset): - def infotodict(seqinfo): + template = create_key(subdir=dataset['bidsDir'],file_suffix=r"run-{{item:02d}}_" + dataset['bidsName'],outtype={outtype}) + return template - info = dict() - shanoir2bids = {shanoir2bids_dict} +def get_dataset_to_key_mapping(shanoir2bids): + dataset_to_key = dict() + for dataset in shanoir2bids: + template = create_bids_key(dataset) + dataset_to_key[dataset['datasetName']] = template + return dataset_to_key - dataset_to_key = get_dataset_to_key_mapping(shanoir2bids) - for seq in seqinfo: - if seq.series_description in dataset_to_key.keys(): - key = dataset_to_key[seq.series_description] - if key in info.keys(): - info[key].append(seq.series_id) - else: - info[key] = [seq.series_id] - # remove run- key if not needed (one run only) - info_final = simplify_runs(info) - return info_final +def simplify_runs(info): + info_final = dict() + for key in info.keys(): + if len(info[key])==1: + new_template = key[0].replace('run-{{item:02d}}_','') + new_key = (new_template, key[1], key[2]) + info_final[new_key] = info[key] + else: + info_final[key] = info[key] + return info_final + +def infotodict(seqinfo): + + info = dict() + shanoir2bids = {shanoir2bids_dict} + + dataset_to_key = get_dataset_to_key_mapping(shanoir2bids) + for seq in seqinfo: + if seq.series_description in dataset_to_key.keys(): + key = dataset_to_key[seq.series_description] + if key in info.keys(): + info[key].append(seq.series_id) + else: + info[key] = [seq.series_id] + # remove run- key if not needed (one run only) + info_final = simplify_runs(info) + return info_final """ with open(path_heuristic_file, "w", encoding="utf-8") as file: @@ -247,10 +249,8 @@ def infotodict(seqinfo): pass - def generate_bids_heuristic_file( shanoir2bids_dict, - path_heuristic_file, output_type='("dicom","nii.gz")', ) -> None: @@ -599,7 +599,7 @@ def download_subject(self, subject_to_search): subject_id = su_id if self.to_automri_format: - subject_id = 'su_' + su_id + subject_id = "su_" + su_id # correct BIDS mapping of the searched dataset bids_seq_mapping = { "datasetName": item["datasetName"], @@ -613,7 +613,9 @@ def download_subject(self, subject_to_search): "bids_session_id" ] = bids_seq_session else: - bids_seq_mapping["bids_session_id"] = None + bids_seq_session = None + + bids_seq_mapping["bids_session_id"] = bids_seq_session bids_mapping.append(bids_seq_mapping) @@ -674,10 +676,17 @@ def download_subject(self, subject_to_search): ) as heuristic_file: # Generate Heudiconv heuristic file from configuration.json mapping if self.to_automri_format: - generate_automri_heuristic_file(bids_mapping,heuristic_file.name, subject_id,self.output_file_type, bids_seq_session) - generate_bids_heuristic_file( - bids_mapping, heuristic_file.name, output_type=self.output_file_type - ) + generate_automri_heuristic_file( + bids_mapping, + heuristic_file.name, + subject_id, + self.output_file_type, + bids_seq_session, + ) + else: + generate_bids_heuristic_file( + bids_mapping, heuristic_file.name, output_type=self.output_file_type + ) with tempfile.NamedTemporaryFile( mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" ) as dcm2niix_config_file: @@ -698,7 +707,9 @@ def download_subject(self, subject_to_search): if self.longitudinal: workflow_params["session"] = bids_seq_session + print('toto') if self.to_automri_format: + print('bubu') workflow_params["bids_options"] = None workflow(**workflow_params) From 03618e345878bd0652d50347cfc3931486627cdb Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 28 May 2024 15:04:07 +0200 Subject: [PATCH 49/86] [FIX]: removed automri support and linting --- shanoir2bids_heudiconv.py | 108 ++------------------------------------ 1 file changed, 3 insertions(+), 105 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index c7c561c..f7a8c48 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -171,84 +171,6 @@ def read_json_config_file(json_file): ) -def generate_automri_heuristic_file( - shanoir2bids_dict, - path_heuristic_file, - subject, - output_type='("dicom","nii.gz")', - session=None, -): - if output_type == "dicom": - outtype = '("dicom",)' - elif output_type == "nifti": - outtype = '("nii.gz",)' - else: - outtype = '("dicom","nii.gz")' - - heuristic = f"""import os - -def create_key( - subdir, - file_suffix, - outtype= ("nii.gz", "dicom"), - annotation_classes= None, - prefix= "", -): - if not subdir: - raise ValueError("subdir must be a valid format string") - template = os.path.join( - prefix, - subdir, - "{subject}_%s" % file_suffix, - ) - return template, outtype, annotation_classes - -def create_bids_key(dataset): - - template = create_key(subdir=dataset['bidsDir'],file_suffix=r"run-{{item:02d}}_" + dataset['bidsName'],outtype={outtype}) - return template - -def get_dataset_to_key_mapping(shanoir2bids): - dataset_to_key = dict() - for dataset in shanoir2bids: - template = create_bids_key(dataset) - dataset_to_key[dataset['datasetName']] = template - return dataset_to_key - -def simplify_runs(info): - info_final = dict() - for key in info.keys(): - if len(info[key])==1: - new_template = key[0].replace('run-{{item:02d}}_','') - new_key = (new_template, key[1], key[2]) - info_final[new_key] = info[key] - else: - info_final[key] = info[key] - return info_final - -def infotodict(seqinfo): - - info = dict() - shanoir2bids = {shanoir2bids_dict} - - dataset_to_key = get_dataset_to_key_mapping(shanoir2bids) - for seq in seqinfo: - if seq.series_description in dataset_to_key.keys(): - key = dataset_to_key[seq.series_description] - if key in info.keys(): - info[key].append(seq.series_id) - else: - info[key] = [seq.series_id] - # remove run- key if not needed (one run only) - info_final = simplify_runs(info) - return info_final - """ - - with open(path_heuristic_file, "w", encoding="utf-8") as file: - file.write(heuristic) - pass - - def generate_bids_heuristic_file( shanoir2bids_dict, path_heuristic_file, @@ -454,9 +376,6 @@ def set_log_filename(self): ) self.log_fn = opj(self.dl_dir, basename) - def switch_to_automri_format(self): - self.to_automri_format = True - def toggle_longitudinal_version(self): self.longitudinal = True @@ -598,8 +517,6 @@ def download_subject(self, subject_to_search): # ID of the subject (sub-*) subject_id = su_id - if self.to_automri_format: - subject_id = "su_" + su_id # correct BIDS mapping of the searched dataset bids_seq_mapping = { "datasetName": item["datasetName"], @@ -675,18 +592,9 @@ def download_subject(self, subject_to_search): mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" ) as heuristic_file: # Generate Heudiconv heuristic file from configuration.json mapping - if self.to_automri_format: - generate_automri_heuristic_file( - bids_mapping, - heuristic_file.name, - subject_id, - self.output_file_type, - bids_seq_session, - ) - else: - generate_bids_heuristic_file( - bids_mapping, heuristic_file.name, output_type=self.output_file_type - ) + generate_bids_heuristic_file( + bids_mapping, heuristic_file.name, output_type=self.output_file_type + ) with tempfile.NamedTemporaryFile( mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" ) as dcm2niix_config_file: @@ -707,10 +615,6 @@ def download_subject(self, subject_to_search): if self.longitudinal: workflow_params["session"] = bids_seq_session - print('toto') - if self.to_automri_format: - print('bubu') - workflow_params["bids_options"] = None workflow(**workflow_params) fp.close() @@ -771,10 +675,6 @@ def main(): help="Toggle longitudinal approach.", ) - parser.add_argument( - "-a", "--automri", action="store_true", help="Switch to automri file tree." - ) - args = parser.parse_args() # Start configuring the DownloadShanoirDatasetToBids class instance @@ -791,8 +691,6 @@ def main(): if args.longitudinal: stb.toggle_longitudinal_version() - if args.automri: - stb.switch_to_automri_format() if not stb.is_correct_dcm2niix(): print( f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}" From 0436e2d3c74f4bdfedd73632893b168563038691 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 3 Jun 2024 11:34:30 +0200 Subject: [PATCH 50/86] [FIX]: removed temporary directories (not supported by python <3.11) by manual creation and cleaning or not if debug --- shanoir2bids_heudiconv.py | 388 ++++++++++++++++++++------------------ 1 file changed, 202 insertions(+), 186 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index f7a8c48..35ca063 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -94,6 +94,14 @@ def banner_msg(msg): If you want to do so, add key "{key}" to you Shanoir2BIDS configuration file :""" +def create_tmp_directory(path_temporary_directory): + tmp_dir = Path(path_temporary_directory) + if tmp_dir.exists(): + shutil.rmtree(tmp_dir) + tmp_dir.mkdir(parents=True) + pass + + def check_date_format(date_to_format): # TRUE FORMAT should be: date_format = 'Y-m-dTH:M:SZ' try: @@ -270,9 +278,10 @@ def __init__(self): self.date_to = None self.longitudinal = False self.to_automri_format = ( - False # Special filenames for automri (close to BIDS format) + False # Special filenames for automri (No longer used ! --> BIDS format) ) self.add_sns = False # Add series number suffix to filename + self.debug_mode = False # No debug mode by default def set_json_config_file(self, json_file): """ @@ -421,203 +430,200 @@ def download_subject(self, subject_to_search): # Real Shanoir2Bids mapping (handle case when solr search term are included) bids_mapping = [] - # temporary directory containing dowloaded DICOM.zip files - with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_dicom: - with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_archive: - # Loop on each sequence defined in the dictionary - for seq in range(self.n_seq): - # Isolate elements that are called many times - shanoir_seq_name = self.shanoir2bids_dict[seq][ - K_DS_NAME - ] # Shanoir sequence name (OLD) - bids_seq_subdir = self.shanoir2bids_dict[seq][ - K_BIDS_DIR - ] # Sequence BIDS subdirectory name (NEW) - bids_seq_name = self.shanoir2bids_dict[seq][ - K_BIDS_NAME - ] # Sequence BIDS nickname (NEW) - if self.longitudinal: - bids_seq_session = self.shanoir2bids_dict[seq][ - K_BIDS_SES - ] # Sequence BIDS nickname (NEW) - - # Print message concerning the sequence that is being downloaded - print( - "\t-", - bids_seq_name, - subject_to_search, - "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", - ) - # Initialize the parser - search_txt = ( - "studyName:" - + self.shanoir_study_id.replace(" ", "?") - + " AND datasetName:" - + shanoir_seq_name.replace(" ", "?") - + " AND subjectName:" - + subject_to_search.replace(" ", "?") - + " AND examinationComment:" - + self.shanoir_session_id.replace(" ", "*") - + " AND examinationDate:[" - + self.date_from - + " TO " - + self.date_to - + "]" + # Manual temporary directories containing dowloaded DICOM.zip and extracted files + # (temporary directories that can be kept are not supported by pythn <3.1 + tmp_dicom = Path(self.dl_dir).joinpath("tmp_dicoms", subject_to_search) + tmp_archive = Path(self.dl_dir).joinpath( + "tmp_archived_dicoms", subject_to_search + ) + create_tmp_directory(tmp_archive) + create_tmp_directory(tmp_dicom) + + # Loop on each sequence defined in the dictionary + for seq in range(self.n_seq): + # Isolate elements that are called many times + shanoir_seq_name = self.shanoir2bids_dict[seq][ + K_DS_NAME + ] # Shanoir sequence name (OLD) + bids_seq_subdir = self.shanoir2bids_dict[seq][ + K_BIDS_DIR + ] # Sequence BIDS subdirectory name (NEW) + bids_seq_name = self.shanoir2bids_dict[seq][ + K_BIDS_NAME + ] # Sequence BIDS nickname (NEW) + if self.longitudinal: + bids_seq_session = self.shanoir2bids_dict[seq][ + K_BIDS_SES + ] # Sequence BIDS nickname (NEW) + + # Print message concerning the sequence that is being downloaded + print( + "\t-", + bids_seq_name, + subject_to_search, + "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", + ) + + # Initialize the parser + search_txt = ( + "studyName:" + + self.shanoir_study_id.replace(" ", "?") + + " AND datasetName:" + + shanoir_seq_name.replace(" ", "?") + + " AND subjectName:" + + subject_to_search.replace(" ", "?") + + " AND examinationComment:" + + self.shanoir_session_id.replace(" ", "*") + + " AND examinationDate:[" + + self.date_from + + " TO " + + self.date_to + + "]" + ) + + args = self.parser.parse_args( + [ + "-u", + self.shanoir_username, + "-d", + self.shanoir_domaine, + "-of", + tmp_archive, + "-em", + "-st", + search_txt, + "-s", + "200", + "-f", + self.shanoir_file_type, + "-so", + "id,ASC", + "-t", + "500", + ] + ) # Increase time out for heavy files + + config = shanoir_downloader.initialize(args) + response = shanoir_downloader.solr_search(config, args) + + # From response, process the data + # Print the number of items found and a list of these items + if response.status_code == 200: + # Invoke shanoir_downloader to download all the data + shanoir_downloader.download_search_results(config, args, response) + + if len(response.json()["content"]) == 0: + warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns +a result on the website. +Search Text : "{}" \n""".format( + search_txt ) + print(warn_msg) + fp.write(warn_msg) + else: + for item in response.json()["content"]: + # Define subject_id + su_id = item["subjectName"] + # If the user has defined a list of edits to subject names... then do the find and replace + for far in self.list_fars: + su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) + # ID of the subject (sub-*) + subject_id = su_id + + # correct BIDS mapping of the searched dataset + bids_seq_mapping = { + "datasetName": item["datasetName"], + "bidsDir": bids_seq_subdir, + "bidsName": bids_seq_name, + "bids_subject_id": subject_id, + } + + if self.longitudinal: + bids_seq_mapping["bids_session_id"] = bids_seq_session + else: + bids_seq_session = None - args = self.parser.parse_args( - [ - "-u", - self.shanoir_username, - "-d", - self.shanoir_domaine, - "-of", - tmp_archive, - "-em", - "-st", - search_txt, - "-s", - "200", - "-f", - self.shanoir_file_type, - "-so", - "id,ASC", - "-t", - "500", - ] - ) # Increase time out for heavy files + bids_seq_mapping["bids_session_id"] = bids_seq_session - config = shanoir_downloader.initialize(args) - response = shanoir_downloader.solr_search(config, args) + bids_mapping.append(bids_seq_mapping) - # From response, process the data - # Print the number of items found and a list of these items - if response.status_code == 200: - # Invoke shanoir_downloader to download all the data - shanoir_downloader.download_search_results( - config, args, response + # Write the information on the data in the log file + fp.write("- datasetId = " + str(item["datasetId"]) + "\n") + fp.write(" -- studyName: " + item["studyName"] + "\n") + fp.write(" -- subjectName: " + item["subjectName"] + "\n") + fp.write(" -- session: " + item["examinationComment"] + "\n") + fp.write(" -- datasetName: " + item["datasetName"] + "\n") + fp.write( + " -- examinationDate: " + item["examinationDate"] + "\n" ) + fp.write(" >> Downloading archive OK\n") + + # Extract the downloaded archive + dl_archive = glob(opj(tmp_archive, "*" + item["id"] + "*.zip"))[ + 0 + ] + with zipfile.ZipFile(dl_archive, "r") as zip_ref: + extraction_dir = opj(tmp_dicom, item["id"]) + zip_ref.extractall(extraction_dir) - if len(response.json()["content"]) == 0: - warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns - a result on the website. - Search Text : "{}" \n""".format( - search_txt - ) - print(warn_msg) - fp.write(warn_msg) - else: - for item in response.json()["content"]: - # Define subject_id - su_id = item["subjectName"] - # If the user has defined a list of edits to subject names... then do the find and replace - for far in self.list_fars: - su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) - # ID of the subject (sub-*) - subject_id = su_id - - # correct BIDS mapping of the searched dataset - bids_seq_mapping = { - "datasetName": item["datasetName"], - "bidsDir": bids_seq_subdir, - "bidsName": bids_seq_name, - "bids_subject_id": subject_id, - } - - if self.longitudinal: - bids_seq_mapping[ - "bids_session_id" - ] = bids_seq_session - else: - bids_seq_session = None - - bids_seq_mapping["bids_session_id"] = bids_seq_session - - bids_mapping.append(bids_seq_mapping) - - # Write the information on the data in the log file - fp.write( - "- datasetId = " + str(item["datasetId"]) + "\n" - ) - fp.write(" -- studyName: " + item["studyName"] + "\n") - fp.write( - " -- subjectName: " + item["subjectName"] + "\n" - ) - fp.write( - " -- session: " + item["examinationComment"] + "\n" - ) - fp.write( - " -- datasetName: " + item["datasetName"] + "\n" - ) - fp.write( - " -- examinationDate: " - + item["examinationDate"] - + "\n" - ) - fp.write(" >> Downloading archive OK\n") - - # Extract the downloaded archive - dl_archive = glob( - opj(tmp_archive, "*" + item["id"] + "*.zip") - )[0] - with zipfile.ZipFile(dl_archive, "r") as zip_ref: - extraction_dir = opj(tmp_dicom, item["id"]) - zip_ref.extractall(extraction_dir) - - fp.write( - " >> Extraction of all files from archive '" - + dl_archive - + " into " - + extraction_dir - + "\n" - ) - - elif response.status_code == 204: - banner_msg("ERROR : No file found!") - fp.write(" >> ERROR : No file found!\n") - else: - banner_msg( - "ERROR : Returned by the request: status of the response = " - + response.status_code - ) fp.write( - " >> ERROR : Returned by the request: status of the response = " - + str(response.status_code) + " >> Extraction of all files from archive '" + + dl_archive + + " into " + + extraction_dir + "\n" ) - # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options - with tempfile.NamedTemporaryFile( - mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" - ) as heuristic_file: - # Generate Heudiconv heuristic file from configuration.json mapping - generate_bids_heuristic_file( - bids_mapping, heuristic_file.name, output_type=self.output_file_type + elif response.status_code == 204: + banner_msg("ERROR : No file found!") + fp.write(" >> ERROR : No file found!\n") + else: + banner_msg( + "ERROR : Returned by the request: status of the response = " + + response.status_code ) - with tempfile.NamedTemporaryFile( - mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" - ) as dcm2niix_config_file: - self.export_dcm2niix_config_options(dcm2niix_config_file.name) - workflow_params = { - "files": glob(opj(tmp_dicom, "*", "*.dcm"), recursive=True), - "outdir": opj(self.dl_dir, str(self.shanoir_study_id)), - "subjs": [subject_id], - "converter": "dcm2niix", - "heuristic": heuristic_file.name, - "bids_options": "--bids", - # "with_prov": True, - "dcmconfig": dcm2niix_config_file.name, - "datalad": True, - "minmeta": True, - "grouping": "all", # other options are too restrictive (tested on EMISEP) - } - - if self.longitudinal: - workflow_params["session"] = bids_seq_session - - workflow(**workflow_params) - fp.close() + fp.write( + " >> ERROR : Returned by the request: status of the response = " + + str(response.status_code) + + "\n" + ) + + # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options + with tempfile.NamedTemporaryFile( + mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" + ) as heuristic_file: + # Generate Heudiconv heuristic file from configuration.json mapping + generate_bids_heuristic_file( + bids_mapping, heuristic_file.name, output_type=self.output_file_type + ) + with tempfile.NamedTemporaryFile( + mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" + ) as dcm2niix_config_file: + self.export_dcm2niix_config_options(dcm2niix_config_file.name) + workflow_params = { + "files": glob(opj(tmp_dicom, "*", "*.dcm"), recursive=True), + "outdir": opj(self.dl_dir, str(self.shanoir_study_id)), + "subjs": [subject_id], + "converter": "dcm2niix", + "heuristic": heuristic_file.name, + "bids_options": "--bids", + # "with_prov": True, + "dcmconfig": dcm2niix_config_file.name, + "datalad": True, + "minmeta": True, + "grouping": "all", # other options are too restrictive (tested on EMISEP) + } + + if self.longitudinal: + workflow_params["session"] = bids_seq_session + + workflow(**workflow_params) + fp.close() + if not self.debug_mode: + shutil.rmtree(tmp_archive) + shutil.rmtree(tmp_dicom) + def download(self): """ @@ -674,6 +680,12 @@ def main(): action="store_true", help="Toggle longitudinal approach.", ) + parser.add_argument( + "--debug", + required=False, + action="store_true", + help="Toggle debug mode (keep temporary directories)", + ) args = parser.parse_args() @@ -689,8 +701,12 @@ def main(): dl_dir=args.output_folder ) # output folder (if None a default directory is created) + if args.debug: + stb.debug_mode = True + if args.longitudinal: stb.toggle_longitudinal_version() + if not stb.is_correct_dcm2niix(): print( f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}" From 36e311fad67f3bb9156c6e246a268544a7637aa9 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 3 Jun 2024 13:29:11 +0200 Subject: [PATCH 51/86] [FIX]: --- shanoir2bids_heudiconv.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index 35ca063..cccb35f 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -489,7 +489,7 @@ def download_subject(self, subject_to_search): "-d", self.shanoir_domaine, "-of", - tmp_archive, + str(tmp_archive), "-em", "-st", search_txt, @@ -621,8 +621,8 @@ def download_subject(self, subject_to_search): workflow(**workflow_params) fp.close() if not self.debug_mode: - shutil.rmtree(tmp_archive) - shutil.rmtree(tmp_dicom) + shutil.rmtree(tmp_archive.parent) + shutil.rmtree(tmp_dicom.parent) def download(self): From f5d5029a1cd0eeda90ae50dffce78cbee71a084e Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Thu, 27 Jun 2024 12:15:59 +0200 Subject: [PATCH 52/86] [ENH]: added temporary directories --- shanoir2bids_heudiconv.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index cccb35f..d1b0159 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -613,6 +613,7 @@ def download_subject(self, subject_to_search): "datalad": True, "minmeta": True, "grouping": "all", # other options are too restrictive (tested on EMISEP) + "overwrite":True } if self.longitudinal: @@ -621,9 +622,11 @@ def download_subject(self, subject_to_search): workflow(**workflow_params) fp.close() if not self.debug_mode: - shutil.rmtree(tmp_archive.parent) - shutil.rmtree(tmp_dicom.parent) - + shutil.rmtree(tmp_archive, ignore_errors=True) + shutil.rmtree(tmp_dicom, ignore_errors=True) + # beware of side effects + shutil.rmtree(tmp_archive.parent, ignore_errors=True) + shutil.rmtree(tmp_dicom.parent, ignore_errors=True) def download(self): """ From 1534a3eba662ad1b437c643dc029f6521aebafc5 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Thu, 27 Jun 2024 16:34:28 +0200 Subject: [PATCH 53/86] [ENH]: check BIDS mapping beforehand --- shanoir2bids_heudiconv.py | 64 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index d1b0159..7f5ce66 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -26,6 +26,7 @@ # import loggger used in heudiconv workflow from heudiconv.main import lgr +import bids_validator # Load environment variables @@ -245,6 +246,7 @@ def infotodict(seqinfo): pass + class DownloadShanoirDatasetToBIDS: """ class that handles the downloading of shanoir data set and the reformatting as a BIDS data structure @@ -281,7 +283,7 @@ def __init__(self): False # Special filenames for automri (No longer used ! --> BIDS format) ) self.add_sns = False # Add series number suffix to filename - self.debug_mode = False # No debug mode by default + self.debug_mode = False # No debug mode by default def set_json_config_file(self, json_file): """ @@ -415,6 +417,57 @@ def configure_parser(self): shanoir_downloader.add_search_arguments(self.parser) shanoir_downloader.add_ids_arguments(self.parser) + def is_mapping_bids(self): + """Check BIDS compliance of filenames/path used in the configuration file""" + validator = bids_validator.BIDSValidator() + + subjects = self.shanoir_subjects + list_find_and_replace = self.list_fars + if list_find_and_replace: + # normalise subjects name + normalised_subjects = [] + for subject in subjects: + for i, far in enumerate(list_find_and_replace): + if i == 0: + normalised_subject = subject + normalised_subject = normalised_subject.replace(far["find"], far["replace"]) + normalised_subjects.append(normalised_subject) + else: + normalised_subjects = subjects + + sessions = self.shanoir_session_id + extension = '.nii.gz' + + if sessions == '*': + paths = ( + "/" + "sub-" + subject + '/' + + map["bidsDir"] + '/' + + "sub-" + subject + '_' + + map["bidsName"] + extension + + for subject in normalised_subjects + for map in self.shanoir2bids_dict + ) + else: + paths = ( + "/" + "sub-" + subject + '/' + + "ses-" + session + '/' + + map["bidsDir"] + '/' + + "sub-" + subject + '_' + "ses-" + session + '_' + + map["bidsName"] + extension + + for session in sessions + for subject in normalised_subjects + for map in self.shanoir2bids_dict + ) + + bids_errors = [p for p in paths if not validator.is_bids(p)] + + if not bids_errors: + return True, bids_errors + else: + return False, bids_errors + def download_subject(self, subject_to_search): """ For a single subject @@ -613,7 +666,7 @@ def download_subject(self, subject_to_search): "datalad": True, "minmeta": True, "grouping": "all", # other options are too restrictive (tested on EMISEP) - "overwrite":True + "overwrite": True, } if self.longitudinal: @@ -715,7 +768,12 @@ def main(): f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}" ) else: - stb.download() + if stb.is_mapping_bids()[0]: + stb.download() + else: + print( + f"Provided BIDS keys {stb.is_mapping_bids()[1]} are not BIDS compliant check syntax in provided configuration file {args.config_file}" + ) if __name__ == "__main__": From ce7019cced4ce7df8f11e87d590f3e9aa0b5886b Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 15 Jul 2024 14:07:50 +0200 Subject: [PATCH 54/86] [ENH]: shanoir2bids_heudiconv is now shanoir2bids --- shanoir2bids.py | 808 +++++++++++++++++++++++++++++------------------- 1 file changed, 487 insertions(+), 321 deletions(-) diff --git a/shanoir2bids.py b/shanoir2bids.py index 50d751d..ecf5545 100755 --- a/shanoir2bids.py +++ b/shanoir2bids.py @@ -4,36 +4,41 @@ The script is made to run for every project given some information provided by the user into a ".json" configuration file. More details regarding the configuration file in the Readme.md""" # Script to download and BIDS-like organize data on Shanoir using "shanoir_downloader.py" developed by Arthur Masson -# @Author: Malo Gaubert , Quentin Duché -# @Date: 24 Juin 2022 + + import os -import sys -import zipfile -import json -import shutil -import shanoir_downloader -from dotenv import load_dotenv -from pathlib import Path from os.path import join as opj, splitext as ops, exists as ope, dirname as opd from glob import glob +import sys +from pathlib import Path from time import time +import zipfile import datetime +import tempfile from dateutil import parser +import json +import logging +import shutil +import shanoir_downloader +from dotenv import load_dotenv +from heudiconv.main import workflow -# TODO list -# 1) Ajouter option pour conserver/supprimer le dicom folder apres conversion dcm2niix +# import loggger used in heudiconv workflow +from heudiconv.main import lgr +import bids_validator # Load environment variables -load_dotenv(dotenv_path=opj(opd(__file__), '.env')) +load_dotenv(dotenv_path=opj(opd(__file__), ".env")) + def banner_msg(msg): """ Print a message framed by a banner of "*" characters :param msg: """ - banner = '*' * (len(msg) + 6) - print(banner + '\n* ', msg, ' *\n' + banner) + banner = "*" * (len(msg) + 6) + print(banner + "\n* ", msg, " *\n" + banner) # Keys for json configuration file @@ -44,30 +49,40 @@ def banner_msg(msg): K_JSON_FIND_AND_REPLACE = "find_and_replace_subject" K_DCM2NIIX_PATH = "dcm2niix" K_DCM2NIIX_OPTS = "dcm2niix_options" -K_FIND = 'find' -K_REPLACE = 'replace' -K_JSON_DATE_FROM = 'date_from' # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] -K_JSON_DATE_TO = 'date_to' # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] +K_FIND = "find" +K_REPLACE = "replace" +K_JSON_DATE_FROM = ( + "date_from" # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] +) +K_JSON_DATE_TO = ( + "date_to" # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] +) LIST_MANDATORY_KEYS_JSON = [K_JSON_STUDY_NAME, K_JSON_L_SUBJECTS, K_JSON_DATA_DICT] -LIST_AUTHORIZED_KEYS_JSON = LIST_MANDATORY_KEYS_JSON + [K_DCM2NIIX_PATH, K_DCM2NIIX_OPTS, K_JSON_DATE_FROM, K_JSON_DATE_TO, K_JSON_SESSION] +LIST_AUTHORIZED_KEYS_JSON = LIST_MANDATORY_KEYS_JSON + [ + K_DCM2NIIX_PATH, + K_DCM2NIIX_OPTS, + K_JSON_DATE_FROM, + K_JSON_DATE_TO, + K_JSON_SESSION, +] # Define keys for data dictionary -K_BIDS_NAME = 'bidsName' -K_BIDS_DIR = 'bidsDir' -K_BIDS_SES = 'bidsSession' -K_DS_NAME = 'datasetName' +K_BIDS_NAME = "bidsName" +K_BIDS_DIR = "bidsDir" +K_BIDS_SES = "bidsSession" +K_DS_NAME = "datasetName" # Define Extensions that are dealt so far by (#todo : think of other possible extensions ?) -NIFTI = '.nii' -NIIGZ = '.nii.gz' -JSON = '.json' -BVAL = '.bval' -BVEC = '.bvec' -DCM = '.dcm' +NIFTI = ".nii" +NIIGZ = ".nii.gz" +JSON = ".json" +BVAL = ".bval" +BVEC = ".bvec" +DCM = ".dcm" # Shanoir parameters -SHANOIR_FILE_TYPE_NIFTI = 'nifti' -SHANOIR_FILE_TYPE_DICOM = 'dicom' +SHANOIR_FILE_TYPE_NIFTI = "nifti" +SHANOIR_FILE_TYPE_DICOM = "dicom" DEFAULT_SHANOIR_FILE_TYPE = SHANOIR_FILE_TYPE_NIFTI # Define error and warning messages when call to dcm2niix is not well configured in the json file @@ -79,13 +94,24 @@ def banner_msg(msg): If you want to do so, add key "{key}" to you Shanoir2BIDS configuration file :""" +def create_tmp_directory(path_temporary_directory): + tmp_dir = Path(path_temporary_directory) + if tmp_dir.exists(): + shutil.rmtree(tmp_dir) + tmp_dir.mkdir(parents=True) + pass + + def check_date_format(date_to_format): # TRUE FORMAT should be: date_format = 'Y-m-dTH:M:SZ' try: parser.parse(date_to_format) # If the date validation goes wrong except ValueError: - print("Incorrect data format, should be YYYY-MM-DDTHH:MM:SSZ (for example: 2020-02-19T00:00:00Z)") + print( + "Incorrect data format, should be YYYY-MM-DDTHH:MM:SSZ (for example: 2020-02-19T00:00:00Z)" + ) + def read_json_config_file(json_file): """ @@ -113,9 +139,9 @@ def read_json_config_file(json_file): list_fars = [] dcm2niix_path = None dcm2niix_opts = None - date_from = '*' - date_to = '*' - session_id = '*' + date_from = "*" + date_to = "*" + session_id = "*" if K_JSON_FIND_AND_REPLACE in data.keys(): list_fars = data[K_JSON_FIND_AND_REPLACE] @@ -124,40 +150,121 @@ def read_json_config_file(json_file): if K_DCM2NIIX_OPTS in data.keys(): dcm2niix_opts = data[K_DCM2NIIX_OPTS] if K_JSON_DATE_FROM in data.keys(): - if data[K_JSON_DATE_FROM] == '': - data_from = '*' + if data[K_JSON_DATE_FROM] == "": + data_from = "*" else: date_from = data[K_JSON_DATE_FROM] check_date_format(date_from) if K_JSON_DATE_TO in data.keys(): - if data[K_JSON_DATE_TO] == '': - data_to = '*' + if data[K_JSON_DATE_TO] == "": + data_to = "*" else: date_to = data[K_JSON_DATE_TO] check_date_format(date_to) if K_JSON_SESSION in data.keys(): session_id = data[K_JSON_SESSION] - # Close json file and return f.close() - return study_id, subjects, session_id, data_dict, list_fars, dcm2niix_path, dcm2niix_opts, date_from, date_to + return ( + study_id, + subjects, + session_id, + data_dict, + list_fars, + dcm2niix_path, + dcm2niix_opts, + date_from, + date_to, + ) + + +def generate_bids_heuristic_file( + shanoir2bids_dict, + path_heuristic_file, + output_type='("dicom","nii.gz")', +) -> None: + """Generate heudiconv heuristic.py file from shanoir2bids mapping dict + Parameters + ---------- + shanoir2bids_dict : + path_heuristic_file : path of the python heuristic file (.py) + """ + if output_type == "dicom": + outtype = '("dicom",)' + elif output_type == "nifti": + outtype = '("nii.gz",)' + else: + outtype = '("dicom","nii.gz")' + + heuristic = f"""from heudiconv.heuristics.reproin import create_key + +def create_bids_key(dataset): + + template = create_key(subdir=dataset['bidsDir'],file_suffix=r"run-{{item:02d}}_" + dataset['bidsName'],outtype={outtype}) + return template + +def get_dataset_to_key_mapping(shanoir2bids): + dataset_to_key = dict() + for dataset in shanoir2bids: + template = create_bids_key(dataset) + dataset_to_key[dataset['datasetName']] = template + return dataset_to_key + +def simplify_runs(info): + info_final = dict() + for key in info.keys(): + if len(info[key])==1: + new_template = key[0].replace('run-{{item:02d}}_','') + new_key = (new_template, key[1], key[2]) + info_final[new_key] = info[key] + else: + info_final[key] = info[key] + return info_final + +def infotodict(seqinfo): + + info = dict() + shanoir2bids = {shanoir2bids_dict} + + dataset_to_key = get_dataset_to_key_mapping(shanoir2bids) + for seq in seqinfo: + if seq.series_description in dataset_to_key.keys(): + key = dataset_to_key[seq.series_description] + if key in info.keys(): + info[key].append(seq.series_id) + else: + info[key] = [seq.series_id] + # remove run- key if not needed (one run only) + info_final = simplify_runs(info) + return info_final +""" + + with open(path_heuristic_file, "w", encoding="utf-8") as file: + file.write(heuristic) + pass class DownloadShanoirDatasetToBIDS: """ class that handles the downloading of shanoir data set and the reformatting as a BIDS data structure """ + def __init__(self): """ Initialize the class instance """ self.shanoir_subjects = None # List of Shanoir subjects - self.shanoir2bids_dict = None # Dictionary specifying how to reformat data into BIDS structure + self.shanoir2bids_dict = ( + None # Dictionary specifying how to reformat data into BIDS structure + ) self.shanoir_username = None # Shanoir username self.shanoir_study_id = None # Shanoir study ID self.shanoir_session_id = None # Shanoir study ID - self.shanoir_file_type = DEFAULT_SHANOIR_FILE_TYPE # Default download type (nifti/dicom) + self.shanoir_file_type = SHANOIR_FILE_TYPE_DICOM # Download File Type (DICOM) + self.output_file_type = ( + DEFAULT_SHANOIR_FILE_TYPE # Default Export File Type (NIFTI) + ) self.json_config_file = None self.list_fars = [] # List of substrings to edit in subjects names self.dl_dir = None # download directory, where data will be stored @@ -165,12 +272,16 @@ def __init__(self): self.n_seq = 0 # Number of sequences in the shanoir2bids_dict self.log_fn = None self.dcm2niix_path = None # Path to the dcm2niix the user wants to use + self.actual_dcm2niix_path = shutil.which("dcm2niix") self.dcm2niix_opts = None # Options to add to the dcm2niix call self.date_from = None self.date_to = None self.longitudinal = False - self.to_automri_format = False # Special filenames for automri (close to BIDS format) - self.add_sns = False # Add series number suffix to filename + self.to_automri_format = ( + False # Special filenames for automri (No longer used ! --> BIDS format) + ) + self.add_sns = False # Add series number suffix to filename + self.debug_mode = False # No debug mode by default def set_json_config_file(self, json_file): """ @@ -178,21 +289,33 @@ def set_json_config_file(self, json_file): :param json_file: str, path to the json_file """ self.json_config_file = json_file - study_id, subjects, session_id, data_dict, list_fars, dcm2niix_path, dcm2niix_opts, date_from, date_to = read_json_config_file(json_file=json_file) + ( + study_id, + subjects, + session_id, + data_dict, + list_fars, + dcm2niix_path, + dcm2niix_opts, + date_from, + date_to, + ) = read_json_config_file(json_file=json_file) self.set_shanoir_study_id(study_id=study_id) self.set_shanoir_subjects(subjects=subjects) self.set_shanoir_session_id(session_id=session_id) self.set_shanoir2bids_dict(data_dict=data_dict) self.set_shanoir_list_find_and_replace(list_fars=list_fars) - self.set_dcm2niix_parameters(dcm2niix_path=dcm2niix_path, dcm2niix_opts=dcm2niix_opts) + self.set_dcm2niix_parameters( + dcm2niix_path=dcm2niix_path, dcm2niix_opts=dcm2niix_opts + ) self.set_date_from(date_from=date_from) self.set_date_to(date_to=date_to) - def set_shanoir_file_type(self, shanoir_file_type): - if shanoir_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI]: - self.shanoir_file_type = shanoir_file_type + def set_output_file_type(self, outfile_type): + if outfile_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, "both"]: + self.output_file_type = outfile_type else: - sys.exit('Unknown shanoir file type {}'.format(shanoir_file_type)) + sys.exit("Unknown output file type {}".format(outfile_type)) def set_shanoir_study_id(self, study_id): self.shanoir_study_id = study_id @@ -200,7 +323,7 @@ def set_shanoir_study_id(self, study_id): def set_shanoir_username(self, shanoir_username): self.shanoir_username = shanoir_username - def set_shanoir_domaine(self, shanoir_domaine): + def set_shanoir_domaine(self, shanoir_domaine): self.shanoir_domaine = shanoir_domaine def set_shanoir_subjects(self, subjects): @@ -216,6 +339,12 @@ def set_dcm2niix_parameters(self, dcm2niix_path, dcm2niix_opts): self.dcm2niix_path = dcm2niix_path self.dcm2niix_opts = dcm2niix_opts + def export_dcm2niix_config_options(self, path_dcm2niix_options_file): + # Serializing json + json_object = json.dumps(self.dcm2niix_opts, indent=4) + with open(path_dcm2niix_options_file, "w") as file: + file.write(json_object) + def set_date_from(self, date_from): self.date_from = date_from @@ -230,8 +359,13 @@ def set_download_directory(self, dl_dir): if dl_dir is None: # Create a default download directory dt = datetime.datetime.now().strftime("%Y_%m_%d_at_%Hh%Mm%Ss") - self.dl_dir = '_'.join(["shanoir2bids", "download", self.shanoir_study_id, dt]) - print('A NEW DEFAULT directory is created as you did not provide a download directory (-of option)\n\t' + self.dl_dir) + self.dl_dir = "_".join( + ["shanoir2bids", "download", self.shanoir_study_id, dt] + ) + print( + "A NEW DEFAULT directory is created as you did not provide a download directory (-of option)\n\t" + + self.dl_dir + ) else: self.dl_dir = dl_dir # Create directory if it does not exist @@ -241,33 +375,97 @@ def set_download_directory(self, dl_dir): def set_log_filename(self): curr_time = datetime.datetime.now() - basename = 'shanoir_downloader_{:04}{:02}{:02}_{:02}{:02}{:02}.log'.format(curr_time.year, - curr_time.month, - curr_time.day, - curr_time.hour, - curr_time.minute, - curr_time.second) + basename = "shanoir_downloader_{:04}{:02}{:02}_{:02}{:02}{:02}.log".format( + curr_time.year, + curr_time.month, + curr_time.day, + curr_time.hour, + curr_time.minute, + curr_time.second, + ) self.log_fn = opj(self.dl_dir, basename) def toggle_longitudinal_version(self): self.longitudinal = True - def switch_to_automri_format(self): - self.to_automri_format = True - - def add_series_number_suffix(self): - self.add_sns = True + def is_correct_dcm2niix(self): + current_version = Path(self.actual_dcm2niix_path) + config_version = Path(self.dcm2niix_path) + if current_version is not None and config_version is not None: + return config_version.samefile(current_version) + else: + return False def configure_parser(self): """ Configure the parser and the configuration of the shanoir_downloader """ self.parser = shanoir_downloader.create_arg_parser() - shanoir_downloader.add_common_arguments(self.parser) + shanoir_downloader.add_username_argument(self.parser) + shanoir_downloader.add_domain_argument(self.parser) + self.parser.add_argument( + "-f", + "--format", + default="dicom", + choices=["dicom"], + help="The format to download.", + ) + shanoir_downloader.add_output_folder_argument(self.parser) shanoir_downloader.add_configuration_arguments(self.parser) shanoir_downloader.add_search_arguments(self.parser) shanoir_downloader.add_ids_arguments(self.parser) + def is_mapping_bids(self): + """Check BIDS compliance of filenames/path used in the configuration file""" + validator = bids_validator.BIDSValidator() + + subjects = self.shanoir_subjects + list_find_and_replace = self.list_fars + if list_find_and_replace: + # normalise subjects name + normalised_subjects = [] + for subject in subjects: + for i, far in enumerate(list_find_and_replace): + if i == 0: + normalised_subject = subject + normalised_subject = normalised_subject.replace(far["find"], far["replace"]) + normalised_subjects.append(normalised_subject) + else: + normalised_subjects = subjects + + sessions = self.shanoir_session_id + extension = '.nii.gz' + + if sessions == '*': + paths = ( + "/" + "sub-" + subject + '/' + + map["bidsDir"] + '/' + + "sub-" + subject + '_' + + map["bidsName"] + extension + + for subject in normalised_subjects + for map in self.shanoir2bids_dict + ) + else: + paths = ( + "/" + "sub-" + subject + '/' + + "ses-" + session + '/' + + map["bidsDir"] + '/' + + "sub-" + subject + '_' + "ses-" + session + '_' + + map["bidsName"] + extension + + for session in sessions + for subject in normalised_subjects + for map in self.shanoir2bids_dict + ) + + bids_errors = [p for p in paths if not validator.is_bids(p)] + + if not bids_errors: + return True, bids_errors + else: + return False, bids_errors + def download_subject(self, subject_to_search): """ For a single subject @@ -279,37 +477,83 @@ def download_subject(self, subject_to_search): banner_msg("Downloading subject " + subject_to_search) # Open log file to write the steps of processing (downloading, renaming...) - fp = open(self.log_fn, 'a') + fp = open(self.log_fn, "a") + + # Real Shanoir2Bids mapping (handle case when solr search term are included) + bids_mapping = [] + + # Manual temporary directories containing dowloaded DICOM.zip and extracted files + # (temporary directories that can be kept are not supported by pythn <3.1 + tmp_dicom = Path(self.dl_dir).joinpath("tmp_dicoms", subject_to_search) + tmp_archive = Path(self.dl_dir).joinpath( + "tmp_archived_dicoms", subject_to_search + ) + create_tmp_directory(tmp_archive) + create_tmp_directory(tmp_dicom) # Loop on each sequence defined in the dictionary for seq in range(self.n_seq): # Isolate elements that are called many times - shanoir_seq_name = self.shanoir2bids_dict[seq][K_DS_NAME] # Shanoir sequence name (OLD) - bids_seq_subdir = self.shanoir2bids_dict[seq][K_BIDS_DIR] # Sequence BIDS subdirectory name (NEW) - bids_seq_name = self.shanoir2bids_dict[seq][K_BIDS_NAME] # Sequence BIDS nickname (NEW) + shanoir_seq_name = self.shanoir2bids_dict[seq][ + K_DS_NAME + ] # Shanoir sequence name (OLD) + bids_seq_subdir = self.shanoir2bids_dict[seq][ + K_BIDS_DIR + ] # Sequence BIDS subdirectory name (NEW) + bids_seq_name = self.shanoir2bids_dict[seq][ + K_BIDS_NAME + ] # Sequence BIDS nickname (NEW) if self.longitudinal: - bids_seq_session = self.shanoir2bids_dict[seq][K_BIDS_SES] # Sequence BIDS nickname (NEW) + bids_seq_session = self.shanoir2bids_dict[seq][ + K_BIDS_SES + ] # Sequence BIDS nickname (NEW) # Print message concerning the sequence that is being downloaded - print('\t-', bids_seq_name, subject_to_search, '[' + str(seq + 1) + '/' + str(self.n_seq) + ']') + print( + "\t-", + bids_seq_name, + subject_to_search, + "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", + ) # Initialize the parser - search_txt = 'studyName:' + self.shanoir_study_id.replace(" ", "?") + \ - ' AND datasetName:' + shanoir_seq_name.replace(" ", "?") + \ - ' AND subjectName:' + subject_to_search.replace(" ", "?") + \ - ' AND examinationComment:' + self.shanoir_session_id.replace(" ", "*") + \ - ' AND examinationDate:[' + self.date_from + ' TO ' + self.date_to + ']' + search_txt = ( + "studyName:" + + self.shanoir_study_id.replace(" ", "?") + + " AND datasetName:" + + shanoir_seq_name.replace(" ", "?") + + " AND subjectName:" + + subject_to_search.replace(" ", "?") + + " AND examinationComment:" + + self.shanoir_session_id.replace(" ", "*") + + " AND examinationDate:[" + + self.date_from + + " TO " + + self.date_to + + "]" + ) args = self.parser.parse_args( - ['-u', self.shanoir_username, - '-d', self.shanoir_domaine, - '-of', self.dl_dir, - '-em', - '-st', search_txt, - '-s', '200', - '-f', self.shanoir_file_type, - '-so', 'id,ASC', - '-t', '500']) # Increase time out for heavy files + [ + "-u", + self.shanoir_username, + "-d", + self.shanoir_domaine, + "-of", + str(tmp_archive), + "-em", + "-st", + search_txt, + "-s", + "200", + "-f", + self.shanoir_file_type, + "-so", + "id,ASC", + "-t", + "500", + ] + ) # Increase time out for heavy files config = shanoir_downloader.initialize(args) response = shanoir_downloader.solr_search(config, args) @@ -317,245 +561,123 @@ def download_subject(self, subject_to_search): # From response, process the data # Print the number of items found and a list of these items if response.status_code == 200: - # Invoke shanoir_downloader to download all the data shanoir_downloader.download_search_results(config, args, response) - if len(response.json()['content']) == 0: + if len(response.json()["content"]) == 0: warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns a result on the website. -Search Text : "{}" \n""".format(search_txt) +Search Text : "{}" \n""".format( + search_txt + ) print(warn_msg) fp.write(warn_msg) else: - # Organize in BIDS like specifications and rename files - for item in response.json()['content']: + for item in response.json()["content"]: # Define subject_id - su_id = item['subjectName'] + su_id = item["subjectName"] # If the user has defined a list of edits to subject names... then do the find and replace for far in self.list_fars: su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) - # ID of the subject (sub-*) - subject_id = 'sub-' + su_id + subject_id = su_id - # Write the information on the data in the log file - fp.write('- datasetId = ' + str(item['datasetId']) + '\n') - fp.write(' -- studyName: ' + item['studyName'] + '\n') - fp.write(' -- subjectName: ' + item['subjectName'] + '\n') - fp.write(' -- session: ' + item['examinationComment'] + '\n') - fp.write(' -- datasetName: ' + item['datasetName'] + '\n') - fp.write(' -- examinationDate: ' + item['examinationDate'] + '\n') - fp.write(' >> Downloading archive OK\n') - - # Subject BIDS directory - if self.to_automri_format: - subject_dir = opj(self.dl_dir, 'su_' + su_id) - else: - subject_dir = opj(self.dl_dir, subject_id) + # correct BIDS mapping of the searched dataset + bids_seq_mapping = { + "datasetName": item["datasetName"], + "bidsDir": bids_seq_subdir, + "bidsName": bids_seq_name, + "bids_subject_id": subject_id, + } - # Prepare BIDS naming if self.longitudinal: - # Insert a session sub-directory - bids_data_dir = opj(subject_dir, bids_seq_session, bids_seq_subdir) - if self.to_automri_format: - bids_data_basename = '_'.join([bids_seq_session, bids_seq_name]) - else: - bids_data_basename = '_'.join([subject_id, bids_seq_session, bids_seq_name]) + bids_seq_mapping["bids_session_id"] = bids_seq_session else: - bids_data_dir = opj(subject_dir, bids_seq_subdir) - if self.to_automri_format: - bids_data_basename = '_'.join([bids_seq_name, su_id]) - else: - bids_data_basename = '_'.join([subject_id, bids_seq_name]) + bids_seq_session = None - # Create temp directory to make sure the directory is empty before - tmp_dir = opj(self.dl_dir, 'temp_archive') - Path(tmp_dir).mkdir(parents=True, exist_ok=True) + bids_seq_mapping["bids_session_id"] = bids_seq_session - # Create the directory of the subject - Path(subject_dir).mkdir(parents=True, exist_ok=True) - # And create the subdirectories (ignore if exists) - Path(bids_data_dir).mkdir(parents=True, exist_ok=True) + bids_mapping.append(bids_seq_mapping) + + # Write the information on the data in the log file + fp.write("- datasetId = " + str(item["datasetId"]) + "\n") + fp.write(" -- studyName: " + item["studyName"] + "\n") + fp.write(" -- subjectName: " + item["subjectName"] + "\n") + fp.write(" -- session: " + item["examinationComment"] + "\n") + fp.write(" -- datasetName: " + item["datasetName"] + "\n") + fp.write( + " -- examinationDate: " + item["examinationDate"] + "\n" + ) + fp.write(" >> Downloading archive OK\n") # Extract the downloaded archive - dl_archive = glob(opj(self.dl_dir, '*' + item['id'] + '*.zip'))[0] - with zipfile.ZipFile(dl_archive, 'r') as zip_ref: - zip_ref.extractall(tmp_dir) - # Get the list of files in the archive - list_unzipped_files = glob(opj(tmp_dir, '*')) - - fp.write(" >> Extraction of all files from archive '" + dl_archive + " into " + tmp_dir + "\n") - - def seq_name(extension, run_num=0): - """ - Returns a BIDS filename with appropriate basename and potential suffix - ext: extension ('.json' or '.nii.gz' or ...) - run_num : int, if 0 no suffix, else adds suffix '_run-1' or '_run-2' etc... - """ - if run_num > 0: - basename = bids_data_basename + '_run-{0}{1}'.format(run_num, extension) - else: - basename = bids_data_basename + extension - return opj(bids_data_dir, basename) - - def check_duplicated_data(list_existing_file_json, file_to_add_json): - """ - For a list of json file, check if a json file already exists meaning. - We check the flags AcquisitionTime and SequenceName to derive equality test. - :param list_existing_file_json: list of existing json files - :param file_to_add_json: - :return: True if one correspondance is found, False otherwise - """ - f = open(file_to_add_json) - data = json.load(f) - - for old_json in list_existing_file_json: - f_old = open(old_json) - data_old = json.load(f_old) - - if (data['AcquisitionTime'] == data_old['AcquisitionTime']) & (data['SequenceName'] == data_old['SequenceName']): - # If one of the json file has the same AcquisitionTime and SequenceName, then it is a duplicated file - f_old.close() - f.close() - return True - - f.close() - return False - - if self.shanoir_file_type == SHANOIR_FILE_TYPE_DICOM: - # Process the DICOM file by calling dcm2niix - if not self.dcm2niix_path: - err_msg = "msg\n- {file}""".format(msg=DCM2NIIX_ERR_MSG, file=self.json_config_file) - sys.exit(err_msg) - if not self.dcm2niix_opts: - warn_msg = "msg\n- {file}""".format(msg=DCM2NIIX_WARN_MSG, file=self.json_config_file) - print(warn_msg) - - dcm_files = glob(opj(tmp_dir, '*' + DCM)) - # Define dcm2niix options - options = ' {opts} -f {basename} -o {out}'.format(opts=self.dcm2niix_opts, basename=bids_data_basename, out=tmp_dir) - cmd = self.dcm2niix_path + options + ' ' + dcm_files[0] - - # Retrieve dcm2niix output and save it to file - info_dcm = os.popen(cmd) - info_dcm = info_dcm.read() - info_dcm = info_dcm.split('\n') - fp.write('[dcm2niix] ' + '\n[dcm2niix] '.join(info_dcm[2:-1]) + '\n') - - # Remove temporary DICOM files - for dcm_file in dcm_files: - os.remove(dcm_file) - - # After the command, the user should have a nifti and json file and extra files - list_unzipped_files = glob(opj(tmp_dir, '*')) - - # Now the DICOM part of the script should be in the same stage as the NIFTI part of the script which is below - - # Reorder list_unzipped_files to have json file first; otherwise, nii file will be ordered and then, json will be scanned. - # if duplicated, json won't be copied while nii is already ordered - tempo_fn, tempo_ext = ops(list_unzipped_files[0]) - if tempo_ext != JSON: - for idx_fn in range(1, len(list_unzipped_files)): - tempo_fn, tempo_ext = ops(list_unzipped_files[idx_fn]) - if tempo_ext == JSON: - temp = list_unzipped_files[0] - list_unzipped_files[0] = list_unzipped_files[idx_fn] - list_unzipped_files[idx_fn] = temp - break - - # By default, the new data to order is not a duplication - duplicated_data = False - - if self.to_automri_format and self.add_sns: - # Update bids_data_basename with Series Number information - for filename in list_unzipped_files: - if filename.endswith(JSON): - f_temp = open(filename) - json_dataset = json.load(f_temp) - series_number_suffix = str(json_dataset['SeriesNumber']) - bids_data_basename = '_'.join([bids_seq_name, su_id, series_number_suffix]) - - # Rename every element in the list of files that was in the archive - for f in list_unzipped_files: # Loop over files in the archive - fp.write(" >> Processing file '" + f + "'\n") - - filename, ext = ops(f) # os.path.splitext to get extension - if ext == NIFTI: - # In special case of a nifti file, gzip it - cmd = "gzip " + r'"{}"'.format(f) # todo : use a lib - os.system(cmd) - # And update variables. Filename as now '.nii.gz' extension - f = filename + NIIGZ - ext = NIIGZ - if ext == '.gz': - # os.path.splitext returns (filename.nii, '.gz') instead of (filename, '.nii.gz') - ext = NIIGZ - - # Let's process and rename the file - # Compare the contents of the associated json file between previous and new file "AcquisitionTime" - list_existing_f_ext = glob(opj(bids_data_dir, bids_data_basename + '*' + ext)) - nf = len(list_existing_f_ext) - - # if files with same bids_data_basename are present in subjects directory, check json files - if (nf > 0) & (not duplicated_data): - for filename_tempo in list_unzipped_files: - fn_tempo, ext_tempo = ops(filename_tempo) - if (ext_tempo == JSON) & (ext == JSON): - list_existing_f_ext_json = glob(opj(bids_data_dir, bids_data_basename + '*json')) - duplicated_data = check_duplicated_data(list_existing_f_ext_json, filename_tempo) - - if duplicated_data: - fp.write(" \n/!\ File already present in the subject's directory. Data will not be used.\n\n") - list_unzipped_files = [] - - if ext in [NIIGZ, JSON, BVAL, BVEC]: - if not duplicated_data: - bids_filename = seq_name(extension=ext, run_num=0) - if nf == 0: - # No previously existing file : perform the renaming - os.rename(f, bids_filename) - fp.write(" >> Renaming to '" + bids_filename + "'\n") - elif nf == 1: - fp.write(' /!\ One similar filename found ! \n') - # One file already existed : give suffices - # the old file gets run-1 suffix - os.rename(bids_filename, seq_name(extension=ext, run_num=1)) - fp.write(" >> Renaming '" + bids_filename + "' to '" + seq_name(extension=ext, run_num=1) + "'\n") - # the new file gets run-2 suffix - os.rename(f, seq_name(extension=ext, run_num=2)) - fp.write(" >> Renaming to '" + seq_name(extension=ext, run_num=2) + "'\n") - else: # nf >= 2 - # At least two files exist, do not touch previous but add the right suffix to new file - os.rename(f, seq_name(extension=ext, run_num=nf + 1)) - fp.write(" >> Renaming to '" + seq_name(extension=ext, run_num=nf + 1) + "'\n") - else: - print('[BIDS format] The extension', ext, 'is not yet dealt. Ask the authors of the script to make an effort.') - fp.write(' >> [BIDS format] The extension' + ext + 'is not yet dealt. Ask the authors of the script to make an effort.\n') - - # Delete temporary directory (remove files before if duplicated) - if duplicated_data: - for f in list_unzipped_files: - filename, ext = ops(f) - if ext == NIFTI: - if os.path.exists(f + '.gz'): os.remove(f + '.gz') - else: - if os.path.exists(f): os.remove(f) - shutil.rmtree(tmp_dir) - fp.write(' >> Deleting temporary dir ' + tmp_dir + '\n') - os.remove(dl_archive) - fp.write(' >> Deleting downloaded archive ' + dl_archive + '\n\n\n') - # End for item in response.json() - # End else (cas ou response.json()['content']) != 0) + dl_archive = glob(opj(tmp_archive, "*" + item["id"] + "*.zip"))[ + 0 + ] + with zipfile.ZipFile(dl_archive, "r") as zip_ref: + extraction_dir = opj(tmp_dicom, item["id"]) + zip_ref.extractall(extraction_dir) + + fp.write( + " >> Extraction of all files from archive '" + + dl_archive + + " into " + + extraction_dir + + "\n" + ) elif response.status_code == 204: - banner_msg('ERROR : No file found!') - fp.write(' >> ERROR : No file found!\n') + banner_msg("ERROR : No file found!") + fp.write(" >> ERROR : No file found!\n") else: - banner_msg('ERROR : Returned by the request: status of the response = ' + response.status_code) - fp.write(' >> ERROR : Returned by the request: status of the response = ' + str(response.status_code) + '\n') - - fp.close() + banner_msg( + "ERROR : Returned by the request: status of the response = " + + response.status_code + ) + fp.write( + " >> ERROR : Returned by the request: status of the response = " + + str(response.status_code) + + "\n" + ) + + # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options + with tempfile.NamedTemporaryFile( + mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" + ) as heuristic_file: + # Generate Heudiconv heuristic file from configuration.json mapping + generate_bids_heuristic_file( + bids_mapping, heuristic_file.name, output_type=self.output_file_type + ) + with tempfile.NamedTemporaryFile( + mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" + ) as dcm2niix_config_file: + self.export_dcm2niix_config_options(dcm2niix_config_file.name) + workflow_params = { + "files": glob(opj(tmp_dicom, "*", "*.dcm"), recursive=True), + "outdir": opj(self.dl_dir, str(self.shanoir_study_id)), + "subjs": [subject_id], + "converter": "dcm2niix", + "heuristic": heuristic_file.name, + "bids_options": "--bids", + # "with_prov": True, + "dcmconfig": dcm2niix_config_file.name, + "datalad": True, + "minmeta": True, + "grouping": "all", # other options are too restrictive (tested on EMISEP) + "overwrite": True, + } + + if self.longitudinal: + workflow_params["session"] = bids_seq_session + + workflow(**workflow_params) + fp.close() + if not self.debug_mode: + shutil.rmtree(tmp_archive, ignore_errors=True) + shutil.rmtree(tmp_dicom, ignore_errors=True) + # beware of side effects + shutil.rmtree(tmp_archive.parent, ignore_errors=True) + shutil.rmtree(tmp_dicom.parent, ignore_errors=True) def download(self): """ @@ -564,13 +686,17 @@ def download(self): """ self.set_log_filename() self.configure_parser() # Configure the shanoir_downloader parser - fp = open(self.log_fn, 'w') + fp = open(self.log_fn, "w") for subject_to_search in self.shanoir_subjects: t_start_subject = time() self.download_subject(subject_to_search=subject_to_search) dur_min = int((time() - t_start_subject) // 60) dur_sec = int((time() - t_start_subject) % 60) - end_msg = 'Downloaded dataset for subject ' + subject_to_search + ' in {}m{}s'.format(dur_min, dur_sec) + end_msg = ( + "Downloaded dataset for subject " + + subject_to_search + + " in {}m{}s".format(dur_min, dur_sec) + ) banner_msg(end_msg) @@ -579,34 +705,74 @@ def main(): parser = shanoir_downloader.create_arg_parser(description=DESCRIPTION) # Use username and output folder arguments from shanoir_downloader shanoir_downloader.add_username_argument(parser) - parser.add_argument('-d', '--domain', default='shanoir.irisa.fr', help='The shanoir domain to query.') - parser.add_argument('-f', '--format', default='nifti', choices=['nifti', 'dicom'], help='The format to download.') + parser.add_argument( + "-d", + "--domain", + default="shanoir.irisa.fr", + help="The shanoir domain to query.", + ) + + parser.add_argument( + "--outformat", + default="both", + choices=["nifti", "dicom", "both"], + help="The format to download.", + ) + shanoir_downloader.add_output_folder_argument(parser=parser, required=False) # Add the argument for the configuration file - parser.add_argument('-j', '--config_file', required=True, help='Path to the .json configuration file specifying parameters for shanoir downloading.') - parser.add_argument('-L', '--longitudinal', required=False, action='store_true', help='Toggle longitudinal approach.') - parser.add_argument('-a', '--automri', action='store_true', help='Switch to automri file tree.') - parser.add_argument('-A', '--add_sns', action='store_true', help='Add series number suffix (compatible with -a)') - # Parse arguments + parser.add_argument( + "-j", + "--config_file", + required=True, + help="Path to the .json configuration file specifying parameters for shanoir downloading.", + ) + parser.add_argument( + "-L", + "--longitudinal", + required=False, + action="store_true", + help="Toggle longitudinal approach.", + ) + parser.add_argument( + "--debug", + required=False, + action="store_true", + help="Toggle debug mode (keep temporary directories)", + ) + args = parser.parse_args() # Start configuring the DownloadShanoirDatasetToBids class instance stb = DownloadShanoirDatasetToBIDS() stb.set_shanoir_username(args.username) stb.set_shanoir_domaine(args.domain) - stb.set_json_config_file(json_file=args.config_file) # path to json configuration file - stb.set_shanoir_file_type(shanoir_file_type=args.format) # Format (dicom or nifti) - stb.set_download_directory(dl_dir=args.output_folder) # output folder (if None a default directory is created) + stb.set_json_config_file( + json_file=args.config_file + ) # path to json configuration file + stb.set_output_file_type(args.outformat) + stb.set_download_directory( + dl_dir=args.output_folder + ) # output folder (if None a default directory is created) + + if args.debug: + stb.debug_mode = True + if args.longitudinal: stb.toggle_longitudinal_version() - if args.automri: - stb.switch_to_automri_format() - if args.add_sns: - if not args.automri: - print('Warning : -A option is only compatible with -a option.') - stb.add_series_number_suffix() - stb.download() + + if not stb.is_correct_dcm2niix(): + print( + f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}" + ) + else: + if stb.is_mapping_bids()[0]: + stb.download() + else: + print( + f"Provided BIDS keys {stb.is_mapping_bids()[1]} are not BIDS compliant check syntax in provided configuration file {args.config_file}" + ) -if __name__ == '__main__': +if __name__ == "__main__": main() From fced7df4b5bc0f3fffd33d50ac8b5e82a31adb7b Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 16 Jul 2024 09:33:12 +0200 Subject: [PATCH 55/86] [ENH]: enhanced documentation for shanoir2bids --- Readme.md | 40 ++++++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/Readme.md b/Readme.md index 0c2ea88..85f7e43 100644 --- a/Readme.md +++ b/Readme.md @@ -16,16 +16,20 @@ Optionally, rename the `.env.example` to `.env` and set the variables (`shanoir_ *See the "Installing DicomAnonymizer" section if you want to use DicomAnonymizer for the dicom anonymization instead of PyDicom.* -## Usage +## How to download Shanoir datasets ? -There are three scripts to download datasets: - - `shanoir_downloader.py` simply downloads datasets from a id, a list of ids, or directly from a [shanoir search as on the shanoir solr search page](https://shanoir.irisa.fr/shanoir-ng/solr-search), - - `shanoir_downloader_check.py` is a more complete tool ; it enables to download datasets (from a csv or excel file containing a list of dataset ids, or directly from a [shanoir search as on the shanoir solr search page](https://shanoir.irisa.fr/shanoir-ng/solr-search)), verify their content, anonymize them and / or encrypt them. - - `shanoir2bids.py` uses `shanoir_downloader.py` to download Shanoir datasets and reorganises them into a BIDS data structure that is specified by the user with a `.json` configuration file. An example of configuration file is provided `s2b_example_config.json`. +There are three scripts to download datasets from a Shanoir instance: -`shanoir_downloader_check.py` creates two files in the output folder: - - `downloaded_datasets.csv` records the successfully downloaded datasets, - - `missing_datasets.csv` records the datasets which could not be downloaded. +1.`shanoir_downloader.py`: downloads datasets from a id, a list of ids, or directly from a [shanoir search as on the shanoir solr search page](https://shanoir.irisa.fr/shanoir-ng/solr-search), + +2.`shanoir_downloader_check.py`: a more complete tool ; it enables to download datasets (from a csv or excel file containing a list of dataset ids, or directly from a [shanoir search as on the shanoir solr search page](https://shanoir.irisa.fr/shanoir-ng/solr-search)), verify their content, anonymize them and / or encrypt them. + +3.`shanoir2bids.py`: Download datasets from Shanoir in DICOM format and convert them into datalad datasets in BIDS format. The conversion is parameterised by a configuration file in `.json` format. + +### `shanoir_downloader_check.py` + - `shanoir_downloader_check.py` creates two files in the output folder: + - `downloaded_datasets.csv` records the successfully downloaded datasets, + - `missing_datasets.csv` records the datasets which could not be downloaded. With those two files, `shanoir_downloader_check.py` is able to resume a download session (the downloading can be interrupted any time, the tool will not redownload datasets which have already been downloaded). @@ -39,7 +43,9 @@ See `python shanoir_downloader_check.py --help` for more information. You might want to skip the anonymization process and the encryption process with the `--skip_anonymization` and `--skip_encryption` arguments respectively (or `-sa` and `-se`). -For `shanoir2bids.py`, a configuration file must be provided to transform a Shanoir dataset into a BIDS dataset. +### `shanoir2bids.py` + +A `.json` configuration file must be provided to transform a Shanoir dataset into a BIDS dataset. ``` -----------------------------[.json configuration file information]------------------------------- This file will tell the script what Shanoir datasets should be downloaded and how the data will be organised. @@ -50,31 +56,33 @@ The dictionary in the json file must have four keys : "data_to_bids": list of dict, each dictionary specifies datasets to download and BIDS format with the following keys : -> "datasetName": str, Shanoir name for the sequence to search -> "bidsDir" : str, BIDS subdirectory sequence name (eg : "anat", "func" or "dwi", ...) - -> "bidsName" : str, BIDS sequence name (eg: "t1w-mprage", "t2-hr", "cusp66-ap-b0", ...) + -> "bidsName" : str, BIDS sequence name (eg: "t1w", "acq-b0_dir-AP", ...) ``` - -An example is provided in the file `s2b_example_config.json`. +Please refer to the [BIDS starter kit](https://bids-standard.github.io/bids-starter-kit/folders_and_files/files.html) +for exhaustive templates of filenames. A BIDS compatible example is provided in the file `s2b_example_config.json`. To download longitudinal data, a key `session` and a new entry `bidsSession` in `data_to_bids` dictionaries should be defined in the JSON configuration files. Of note, only one session can be downloaded at once. Then, the key `session` is just a string, not a list as for subjects. -### Example usage - +### Download Examples +#### Raw download To download datasets, verify the content of them, anonymize them and / or encrypt them you can use a command like: `python shanoir_downloader_check.py -u username -d shanoir.irisa.fr -ids path/to/datasets_to_download.csv -of path/to/output/folder/ -se -lf path/to/downloads.log` The `example_input_check.csv` file in this repository is an example input file (the format of the `datasets_to_download.csv` file should be the same). +#### Solr search download You can also download datasets from a [SolR search](https://shanoir.irisa.fr/shanoir-ng/solr-search) as on the website: `python shanoir_downloader.py -u amasson -d shanoir.irisa.fr -of /data/amasson/test/shanoir_test4 --search_text "FLAIR" -p 1 -s 2 ` where `--search_text` is the string you would use on [the SolR search page](https://shanoir.irisa.fr/shanoir-ng/solr-search) (for example `(subjectName:(CT* OR demo*) AND studyName:etude test) OR datasetName:*flair*`). More information on the info box of the SolR search page. -`python shanoir2bids.py -c s2b_example_config.json -d my_download_dir` will download Shanoir files identified in the configuration file and saves them as a BIDS data structure into `my_download_dir` +#### BIDS download +`python shanoir2bids.py -j s2b_example_config.json -of my_download_dir --outformat nifti` will download Shanoir datasets identified in the configuration file saves them as DICOM and convert them into a BIDS datalad dataset into `my_download_dir`. -### Search usage +## About Solr Search The `--search_text` and `--expert_mode` arguments work as on the [Shanoir search page](https://shanoir.irisa.fr/shanoir-ng/solr-search). From 714720d7fe6f52ece83cc7c5bd927bd2de682dce Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 16 Jul 2024 10:01:03 +0200 Subject: [PATCH 56/86] [ENH]: added installation instructions and metadata in a pyproject.toml file --- pyproject.toml | 43 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..bd3d52a --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,43 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "shanoir-downloader" +version = "0.0.1" +dynamic = ["version"] +dynamic = ["dependencies", "optional-dependencies"] + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} +optional-dependencies = {dev = { file = ["requirements-dev.txt"] }} + +requires-python = ">=3.8 <3.12" + +authors = [ + {name = "Arthur Masson"}, + {name = "Quentin Duché"}, + {name = "Malo Gaubert"}, +] +maintainers = [ + {name = "Quentin Duché", email = "quentin.duche@inria.fr"} +] +description = "Download Shanoir Datasets in various format using Python" +readme = "README.md" +license = {file = "LICENSE"} +keywords = ["Shanoir", "DICOM", "NIFTI", "BIDS"] +classifiers = [ + "Programming Language :: Python" +] + +[project.urls] +Repository = "https://github.com/Inria-Empenn/shanoir_downloader.git" +"Bug Tracker" = "https://github.com/Inria-Empenn/shanoir_downloader/issues" +Changelog = "https://github.com/Inria-Empenn/shanoir_downloader/blob/master/CHANGELOG.md" + +[project.scripts] +spam-cli = "shanoir-downloader:main_cli" + + +[project.entry-points."spam.magical"] +tomatoes = "spam:main_tomatoes" \ No newline at end of file From eb348a6422692f317199fafb565ac163bec17713 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 16 Jul 2024 10:47:25 +0200 Subject: [PATCH 57/86] [ENH]: fixed dependencies and build backend issue --- pyproject.toml | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index bd3d52a..37a6d78 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,14 +5,10 @@ build-backend = "setuptools.build_meta" [project] name = "shanoir-downloader" version = "0.0.1" -dynamic = ["version"] -dynamic = ["dependencies", "optional-dependencies"] -[tool.setuptools.dynamic] -dependencies = {file = ["requirements.txt"]} -optional-dependencies = {dev = { file = ["requirements-dev.txt"] }} +dynamic = ["dependencies", "optional-dependencies"] -requires-python = ">=3.8 <3.12" +requires-python = ">=3.8, <3.12" authors = [ {name = "Arthur Masson"}, @@ -35,9 +31,16 @@ Repository = "https://github.com/Inria-Empenn/shanoir_downloader.git" "Bug Tracker" = "https://github.com/Inria-Empenn/shanoir_downloader/issues" Changelog = "https://github.com/Inria-Empenn/shanoir_downloader/blob/master/CHANGELOG.md" -[project.scripts] -spam-cli = "shanoir-downloader:main_cli" +[tool.setuptools] +py-modules = [] + +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} +#optional-dependencies = {dev = { file = ["requirements-dev.txt"] }} +#[project.scripts] +#spam-cli = "shanoir-downloader:main_cli" -[project.entry-points."spam.magical"] -tomatoes = "spam:main_tomatoes" \ No newline at end of file +# +#[project.entry-points."spam.magical"] +#tomatoes = "spam:main_tomatoes" \ No newline at end of file From b255d2e0b2b9b836d0a3daf2a0b6389afbcef00d Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 16 Jul 2024 10:47:40 +0200 Subject: [PATCH 58/86] [ENH]: fixed dependencies and build backend issue --- Readme.md => README.md | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename Readme.md => README.md (100%) diff --git a/Readme.md b/README.md similarity index 100% rename from Readme.md rename to README.md From b39ef9a0d1e5e0349ceace54e84636451256926c Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 16 Jul 2024 10:51:47 +0200 Subject: [PATCH 59/86] [ENH]: added authors emails --- pyproject.toml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 37a6d78..5f8e986 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,12 +11,12 @@ dynamic = ["dependencies", "optional-dependencies"] requires-python = ">=3.8, <3.12" authors = [ - {name = "Arthur Masson"}, - {name = "Quentin Duché"}, - {name = "Malo Gaubert"}, + {name = "Arthur Masson",email = "arthur.masson@inria.fr"}, + {name = "Quentin Duché", email = "quentin.duche@irisa.fr"}, + {name = "Malo Gaubert", email = "malo.gaubert@irisa.fr"}, ] maintainers = [ - {name = "Quentin Duché", email = "quentin.duche@inria.fr"} + {name = "Quentin Duché", email = "quentin.duche@irisa.fr"} ] description = "Download Shanoir Datasets in various format using Python" readme = "README.md" From 85f1efa9de423167c0a4e21b28fbc4610187599a Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 16 Jul 2024 10:53:40 +0200 Subject: [PATCH 60/86] [ENH]: adapted installations instructions in the README to new mechanism --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 85f7e43..64fc8ae 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ The scripts `convert_dicoms_to_niftis.py` and `create_previews.py` enable to con It is advised to install the project in [a virtual environment](https://docs.python.org/3/tutorial/venv.html). -[Install python with pip](https://www.python.org/downloads/) ; then use `pip install -r requirements.txt` to install python dependencies. +[Install python with pip](https://www.python.org/downloads/) ; then use `pip install .` to install python dependencies. Optionally, rename the `.env.example` to `.env` and set the variables (`shanoir_password`, `gpg_recipient`) to your needs. From 894a6944db091aeb82d587f5b4722ea06b727765 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 16 Jul 2024 11:27:12 +0200 Subject: [PATCH 61/86] [ENH]: added CI to test shanoir downloader basic installation --- .github/workflows/python-installation.yml | 36 +++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/python-installation.yml diff --git a/.github/workflows/python-installation.yml b/.github/workflows/python-installation.yml new file mode 100644 index 0000000..8263f98 --- /dev/null +++ b/.github/workflows/python-installation.yml @@ -0,0 +1,36 @@ +name: Python installation + +on: [push] + +jobs: + build: + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + python-version: ["3.8", "3.9", "3.10", "3.11"] + #TODO: use github action to read python version from pyproject.toml + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Setup Python # Set Python version + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + # Create virtual env under each python version + # (optional and a bit redundant because we are already on a specific python) + # it insures installation instruction provided in README are working + - name: Install dependencies + run: | + python -m venv test-env --clear + source test-env/bin/activate + python -m pip install --upgrade pip + python -m pip install -e . +# - name: Test with pytest +# run: pytest tests.py --doctest-modules --junitxml=junit/test-results-${{ matrix.python-version }}.xml +# - name: Upload pytest test results +# uses: actions/upload-artifact@v4 +# with: +# name: pytest-results-${{ matrix.python-version }} +# path: junit/test-results-${{ matrix.python-version }}.xml +# # Use always() to always run this step to publish test results when there are test failures +# if: ${{ always() }} From 53b3dd68418225d93c2737ea8cb5a64f3e2b378d Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 16 Jul 2024 11:40:40 +0200 Subject: [PATCH 62/86] [ENH]: restrained to ubuntu in the first time --- .github/workflows/python-installation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/python-installation.yml b/.github/workflows/python-installation.yml index 8263f98..da42404 100644 --- a/.github/workflows/python-installation.yml +++ b/.github/workflows/python-installation.yml @@ -6,7 +6,7 @@ jobs: build: strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] python-version: ["3.8", "3.9", "3.10", "3.11"] #TODO: use github action to read python version from pyproject.toml runs-on: ${{ matrix.os }} From b9a5969041a34b9a00b079631ed8c5a3058cab81 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 16 Jul 2024 11:53:04 +0200 Subject: [PATCH 63/86] [BF]: try fix bug with python 3.11 installation --- pyproject.toml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 5f8e986..909ff39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,5 @@ [build-system] -requires = ["setuptools"] +requires = ["setuptools>=64", "setuptools_scm>=8"] build-backend = "setuptools.build_meta" [project] @@ -34,9 +34,13 @@ Changelog = "https://github.com/Inria-Empenn/shanoir_downloader/blob/master/CHAN [tool.setuptools] py-modules = [] +[tool.setuptools_scm] +version_file = "pkg/_version.py" + [tool.setuptools.dynamic] dependencies = {file = ["requirements.txt"]} + #optional-dependencies = {dev = { file = ["requirements-dev.txt"] }} #[project.scripts] #spam-cli = "shanoir-downloader:main_cli" From 758dcf09511b786cb47482f3c9027ae98c52f410 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 16 Jul 2024 11:54:26 +0200 Subject: [PATCH 64/86] [BF]: fix syntax --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 909ff39..af25598 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -35,7 +35,7 @@ Changelog = "https://github.com/Inria-Empenn/shanoir_downloader/blob/master/CHAN py-modules = [] [tool.setuptools_scm] -version_file = "pkg/_version.py" + [tool.setuptools.dynamic] dependencies = {file = ["requirements.txt"]} From 7f04e6b815fc4411ffbdd7a8a433a104986303ee Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 16 Jul 2024 12:03:51 +0200 Subject: [PATCH 65/86] [BF]: restrained python version to python<3.11 --- .github/workflows/python-installation.yml | 2 +- pyproject.toml | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/python-installation.yml b/.github/workflows/python-installation.yml index da42404..47530d9 100644 --- a/.github/workflows/python-installation.yml +++ b/.github/workflows/python-installation.yml @@ -7,7 +7,7 @@ jobs: strategy: matrix: os: [ubuntu-latest] - python-version: ["3.8", "3.9", "3.10", "3.11"] + python-version: ["3.8", "3.9", "3.10"] #TODO: use github action to read python version from pyproject.toml runs-on: ${{ matrix.os }} steps: diff --git a/pyproject.toml b/pyproject.toml index af25598..9ce4885 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ version = "0.0.1" dynamic = ["dependencies", "optional-dependencies"] -requires-python = ">=3.8, <3.12" +requires-python = ">=3.8, <3.11" authors = [ {name = "Arthur Masson",email = "arthur.masson@inria.fr"}, @@ -36,7 +36,6 @@ py-modules = [] [tool.setuptools_scm] - [tool.setuptools.dynamic] dependencies = {file = ["requirements.txt"]} From 2c990289401ee3cca9ce9d9a26f6bfb9515bf6f6 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Thu, 18 Jul 2024 15:11:52 +0200 Subject: [PATCH 66/86] [BF]: added environment.yml corresponding to requirements.txt --- environment.yml | 208 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 environment.yml diff --git a/environment.yml b/environment.yml new file mode 100644 index 0000000..2465de7 --- /dev/null +++ b/environment.yml @@ -0,0 +1,208 @@ +name: test-install +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - annexremote=1.6.5=py310hff52083_0 + - atk-1.0=2.36.0=h516909a_2 + - backports=1.0=pyhd8ed1ab_3 + - backports.tarfile=1.0.0=pyhd8ed1ab_1 + - boto3=1.34.144=pyhd8ed1ab_0 + - botocore=1.34.144=pyge310_1234567_0 + - brotli=1.1.0=hd590300_1 + - brotli-bin=1.1.0=hd590300_1 + - brotli-python=1.0.9=py310hd8f1fbe_7 + - bz2file=0.98=py_0 + - bzip2=1.0.8=h5eee18b_6 + - c-ares=1.32.2=h4bc722e_0 + - ca-certificates=2024.7.4=hbcca054_0 + - cached-property=1.5.2=hd8ed1ab_1 + - cached_property=1.5.2=pyha770c72_1 + - cairo=1.16.0=hb05425b_5 + - cffi=1.16.0=py310h2fee648_0 + - chardet=5.2.0=py310hff52083_1 + - ci-info=0.3.0=pyhd8ed1ab_0 + - click=8.1.7=unix_pyh707e725_0 + - colorama=0.4.6=pyhd8ed1ab_0 + - cryptography=42.0.8=py310hb1bd9d3_0 + - curl=8.7.1=hdbd6064_0 + - cycler=0.12.1=pyhd8ed1ab_0 + - datalad=1.1.1=py310hff52083_0 + - dbus=1.13.6=he372182_0 + - dcm2niix=1.0.20211006=h4bd325d_0 + - dcmstack=0.8.0=pyhd8ed1ab_3 + - distro=1.9.0=pyhd8ed1ab_0 + - etelemetry=0.3.1=pyhd8ed1ab_0 + - exifread=3.0.0=pyhd8ed1ab_0 + - expat=2.6.2=h59595ed_0 + - fasteners=0.17.3=pyhd8ed1ab_0 + - filelock=3.15.4=pyhd8ed1ab_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.1=h4c34cd2_2 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.53.1=py310h5b4e0ec_0 + - freetype=2.10.4=h0708190_1 + - fribidi=1.0.10=h36c2ea0_0 + - gdk-pixbuf=2.42.10=h5eee18b_0 + - gettext=0.21.1=h27087fc_0 + - git=2.33.0=pl5321hf874766_1 + - git-annex=10.20230626=nodep_h1234567_1 + - glib=2.69.1=he621ea3_2 + - gobject-introspection=1.72.0=py310hbb6d50b_2 + - graphite2=1.3.14=h295c915_1 + - graphviz=2.50.0=h3cd0ef9_0 + - gtk2=2.24.33=h73c1081_2 + - gts=0.7.6=h08bb679_0 + - h2=4.1.0=pyhd8ed1ab_0 + - h5py=3.7.0=nompi_py310h06dffec_100 + - harfbuzz=4.3.0=hf52aaf7_2 + - hdf5=1.12.1=h2b7332f_3 + - heudiconv=1.1.6=pyhd8ed1ab_0 + - hpack=4.0.0=pyh9f0ad1d_0 + - humanize=4.10.0=pyhd8ed1ab_0 + - hyperframe=6.0.1=pyhd8ed1ab_0 + - icu=73.1=h6a678d5_0 + - importlib_metadata=8.0.0=hd8ed1ab_0 + - importlib_resources=6.4.0=pyhd8ed1ab_0 + - iso8601=2.1.0=pyhd8ed1ab_0 + - isodate=0.6.1=pyhd8ed1ab_0 + - jaraco.classes=3.4.0=pyhd8ed1ab_1 + - jaraco.context=5.3.0=pyhd8ed1ab_1 + - jaraco.functools=4.0.0=pyhd8ed1ab_0 + - jeepney=0.8.0=pyhd8ed1ab_0 + - jmespath=1.0.1=pyhd8ed1ab_0 + - jpeg=9e=h0b41bf4_3 + - keyring=25.2.1=pyha804496_0 + - keyrings.alt=5.0.1=pyhd8ed1ab_1 + - kiwisolver=1.4.2=py310hbf28c38_1 + - krb5=1.20.1=h143b758_1 + - lcms2=2.15=hfd0df8a_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h9c3ff4c_0 + - libblas=3.9.0=20_linux64_openblas + - libbrotlicommon=1.1.0=hd590300_1 + - libbrotlidec=1.1.0=hd590300_1 + - libbrotlienc=1.1.0=hd590300_1 + - libcblas=3.9.0=20_linux64_openblas + - libcurl=8.7.1=h251f7ec_0 + - libdeflate=1.17=h0b41bf4_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=hd590300_2 + - libexpat=2.6.2=h59595ed_0 + - libffi=3.4.4=h6a678d5_1 + - libgcc-ng=14.1.0=h77fa898_0 + - libgd=2.3.3=h695aa2c_1 + - libgfortran-ng=14.1.0=h69a702a_0 + - libgfortran5=14.1.0=hc5f4f2c_0 + - libgomp=14.1.0=h77fa898_0 + - libiconv=1.17=hd590300_2 + - liblapack=3.9.0=20_linux64_openblas + - libnghttp2=1.57.0=h2d74bed_0 + - libopenblas=0.3.25=pthreads_h413a1c8_0 + - libpng=1.6.39=h5eee18b_0 + - librsvg=2.54.4=h36cc946_3 + - libssh2=1.10.0=ha35d2d1_2 + - libstdcxx-ng=11.2.0=h1234567_1 + - libtiff=4.5.1=h6a678d5_0 + - libtool=2.4.7=h27087fc_0 + - libuuid=1.41.5=h5eee18b_0 + - libwebp-base=1.4.0=hd590300_0 + - libxcb=1.16=hd590300_0 + - libxcrypt=4.4.36=hd590300_1 + - libxml2=2.10.4=hfdd30dd_2 + - libxslt=1.1.37=h873f0b0_0 + - looseversion=1.3.0=pyhd8ed1ab_0 + - lxml=4.9.1=py310h5764c6d_0 + - lz4-c=1.9.4=h6a678d5_1 + - matplotlib-base=3.5.2=py310h5701ce4_0 + - more-itertools=10.3.0=pyhd8ed1ab_0 + - msgpack-python=1.0.3=py310hbf28c38_1 + - munkres=1.1.4=pyh9f0ad1d_0 + - mutagen=1.47.0=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - networkx=3.2=pyhd8ed1ab_0 + - nipype=1.8.6=py310hff52083_0 + - numpy=1.22.3=py310h4ef5377_2 + - openjpeg=2.4.0=h9ca470c_2 + - openssl=3.3.1=h4bc722e_2 + - p7zip=16.02=h9c3ff4c_1001 + - packaging=24.1=pyhd8ed1ab_0 + - pango=1.50.7=h05da053_0 + - pathlib=1.0.1=py310hff52083_7 + - patool=2.3.0=pyhd8ed1ab_0 + - pcre=8.45=h9c3ff4c_0 + - pcre2=10.37=h032f7d1_0 + - perl=5.32.1=7_hd590300_perl5 + - pip=24.0=py310h06a4308_0 + - pixman=0.40.0=h36c2ea0_0 + - platformdirs=4.2.2=pyhd8ed1ab_0 + - prov=2.0.0=pyhd3deb0d_0 + - psutil=6.0.0=py310hc51659f_0 + - pthread-stubs=0.4=h36c2ea0_1001 + - pycparser=2.22=pyhd8ed1ab_0 + - pydot=3.0.1=py310hff52083_0 + - pyparsing=3.1.2=pyhd8ed1ab_0 + - pyperclip=1.8.2=pyhd8ed1ab_2 + - pysocks=1.7.1=pyha2e5f31_6 + - python=3.10.14=h955ad1f_1 + - python-gitlab=4.8.0=pyhd8ed1ab_0 + - python_abi=3.10=2_cp310 + - rdflib=7.0.0=pyhd8ed1ab_0 + - readline=8.2=h5eee18b_0 + - requests-ftp=0.3.1=py_1 + - requests-toolbelt=1.0.0=pyhd8ed1ab_0 + - s3transfer=0.10.2=pyhd8ed1ab_0 + - secretstorage=3.3.3=py310hff52083_2 + - setuptools=69.5.1=py310h06a4308_0 + - simplejson=3.19.2=py310h2372a71_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.45.3=h5eee18b_0 + - tk=8.6.14=h39e8969_0 + - traits=6.3.2=py310h5764c6d_1 + - typing_extensions=4.12.2=pyha770c72_0 + - tzdata=2024a=h04d1e81_0 + - unicodedata2=15.1.0=py310h2372a71_0 + - wheel=0.43.0=py310h06a4308_0 + - whoosh=2.7.4=py310hff52083_8 + - xorg-libxau=1.0.11=hd590300_0 + - xorg-libxdmcp=1.1.3=h7f98852_0 + - xvfbwrapper=0.2.9=pyhd8ed1ab_1005 + - xz=5.4.6=h5eee18b_1 + - zlib=1.2.13=h5eee18b_1 + - zstandard=0.22.0=py310h1275a96_0 + - zstd=1.5.5=hc292b87_2 + - pip: + - certifi==2021.10.8 + - charset-normalizer==2.0.8 + - dicom2nifti==2.3.0 + - dicomanonymizer==0.0.1 + - idna==3.3 + - importlib-metadata==4.8.2 + - multivolumefile==0.2.3 + - nibabel==5.2.1 + - pandas==1.4.2 + - pillow==9.0.0 + - py7zr==0.17.0 + - pybcj==0.5.0 + - pycryptodomex==3.12.0 + - pydicom==2.2.2 + - pyppmd==0.17.3 + - python-dateutil==2.8.2 + - python-dotenv==0.19.2 + - pytz==2021.3 + - pyzstd==0.15.0 + - requests==2.26.0 + - scipy==1.11.4 + - simpleitk==2.1.1.2 + - texttable==1.6.4 + - tqdm==4.62.3 + - typing-extensions==4.0.1 + - urllib3==1.26.7 + - zipp==3.6.0 +prefix: /home/alpron/softs/miniconda3/envs/test From 9de8f02474349e943238a5d3c8f40b8c0fcf7c49 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Thu, 18 Jul 2024 15:22:37 +0200 Subject: [PATCH 67/86] [BF]: fix pip and conda installation syntax --- .github/workflows/conda-installation.yml | 34 +++++++++++++++++++ ...-installation.yml => pip-installation.yml} | 0 2 files changed, 34 insertions(+) create mode 100644 .github/workflows/conda-installation.yml rename .github/workflows/{python-installation.yml => pip-installation.yml} (100%) diff --git a/.github/workflows/conda-installation.yml b/.github/workflows/conda-installation.yml new file mode 100644 index 0000000..132af25 --- /dev/null +++ b/.github/workflows/conda-installation.yml @@ -0,0 +1,34 @@ +name: Python installation + +on: [push] + +jobs: + build: + strategy: + matrix: + os: [ubuntu-latest] + python-version: ["3.8", "3.9", "3.10"] + #TODO: use github action to read python version from pyproject.toml + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - name: Setup Python + uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + environment-file: environment.yml + activate-environment: test-install + python-version: ${{ matrix.python-version }} + + # Create virtual env under each python version + # (optional and a bit redundant because we are already on a specific python) + # it insures installation instruction provided in README are working +# - name: Test with pytest +# run: pytest tests.py --doctest-modules --junitxml=junit/test-results-${{ matrix.python-version }}.xml +# - name: Upload pytest test results +# uses: actions/upload-artifact@v4 +# with: +# name: pytest-results-${{ matrix.python-version }} +# path: junit/test-results-${{ matrix.python-version }}.xml +# # Use always() to always run this step to publish test results when there are test failures +# if: ${{ always() }} diff --git a/.github/workflows/python-installation.yml b/.github/workflows/pip-installation.yml similarity index 100% rename from .github/workflows/python-installation.yml rename to .github/workflows/pip-installation.yml From 2c181910ccd065e55b1efccb808289c47a1a2482 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Thu, 18 Jul 2024 15:34:16 +0200 Subject: [PATCH 68/86] [BF]: fix pip and conda installation syntax --- .github/workflows/conda-installation.yml | 2 +- .github/workflows/pip-installation.yml | 4 +- README.md | 47 +++++++++++------------- environment.yml | 2 +- 4 files changed, 26 insertions(+), 29 deletions(-) diff --git a/.github/workflows/conda-installation.yml b/.github/workflows/conda-installation.yml index 132af25..d8becbc 100644 --- a/.github/workflows/conda-installation.yml +++ b/.github/workflows/conda-installation.yml @@ -17,7 +17,7 @@ jobs: with: auto-update-conda: true environment-file: environment.yml - activate-environment: test-install + activate-environment: working-env python-version: ${{ matrix.python-version }} # Create virtual env under each python version diff --git a/.github/workflows/pip-installation.yml b/.github/workflows/pip-installation.yml index 47530d9..50c97b9 100644 --- a/.github/workflows/pip-installation.yml +++ b/.github/workflows/pip-installation.yml @@ -21,8 +21,8 @@ jobs: # it insures installation instruction provided in README are working - name: Install dependencies run: | - python -m venv test-env --clear - source test-env/bin/activate + python -m venv working-env --clear + source working-env/bin/activate python -m pip install --upgrade pip python -m pip install -e . # - name: Test with pytest diff --git a/README.md b/README.md index 64fc8ae..e0c44ab 100644 --- a/README.md +++ b/README.md @@ -8,28 +8,29 @@ The scripts `convert_dicoms_to_niftis.py` and `create_previews.py` enable to con ## Install -It is advised to install the project in [a virtual environment](https://docs.python.org/3/tutorial/venv.html). - +It is advised to install the project in python virtual environment relying either on [venv](https://docs.python.org/3/tutorial/venv.html) or preferably [(mini)conda](https://conda.io/projects/conda/en/latest/user-guide/tasks/manage-environments.html) +### Installation with pip [Install python with pip](https://www.python.org/downloads/) ; then use `pip install .` to install python dependencies. Optionally, rename the `.env.example` to `.env` and set the variables (`shanoir_password`, `gpg_recipient`) to your needs. *See the "Installing DicomAnonymizer" section if you want to use DicomAnonymizer for the dicom anonymization instead of PyDicom.* -## How to download Shanoir datasets ? - -There are three scripts to download datasets from a Shanoir instance: - -1.`shanoir_downloader.py`: downloads datasets from a id, a list of ids, or directly from a [shanoir search as on the shanoir solr search page](https://shanoir.irisa.fr/shanoir-ng/solr-search), +### Installation with conda +> [!IMPORTANT] +> This installation is required if shanoir2bids.py is used -2.`shanoir_downloader_check.py`: a more complete tool ; it enables to download datasets (from a csv or excel file containing a list of dataset ids, or directly from a [shanoir search as on the shanoir solr search page](https://shanoir.irisa.fr/shanoir-ng/solr-search)), verify their content, anonymize them and / or encrypt them. +After installing a (mini)conda distribution type in a terminal `conda create -f environment.yml` then activate the environment using `conda activate working-env` where `working-env` is the default name of the created environment +## Usage -3.`shanoir2bids.py`: Download datasets from Shanoir in DICOM format and convert them into datalad datasets in BIDS format. The conversion is parameterised by a configuration file in `.json` format. +There are three scripts to download datasets: + - `shanoir_downloader.py` simply downloads datasets from a id, a list of ids, or directly from a [shanoir search as on the shanoir solr search page](https://shanoir.irisa.fr/shanoir-ng/solr-search), + - `shanoir_downloader_check.py` is a more complete tool ; it enables to download datasets (from a csv or excel file containing a list of dataset ids, or directly from a [shanoir search as on the shanoir solr search page](https://shanoir.irisa.fr/shanoir-ng/solr-search)), verify their content, anonymize them and / or encrypt them. + - `shanoir2bids.py` uses `shanoir_downloader.py` to download Shanoir datasets and reorganises them into a BIDS data structure that is specified by the user with a `.json` configuration file. An example of configuration file is provided `s2b_example_config.json`. -### `shanoir_downloader_check.py` - - `shanoir_downloader_check.py` creates two files in the output folder: - - `downloaded_datasets.csv` records the successfully downloaded datasets, - - `missing_datasets.csv` records the datasets which could not be downloaded. +`shanoir_downloader_check.py` creates two files in the output folder: + - `downloaded_datasets.csv` records the successfully downloaded datasets, + - `missing_datasets.csv` records the datasets which could not be downloaded. With those two files, `shanoir_downloader_check.py` is able to resume a download session (the downloading can be interrupted any time, the tool will not redownload datasets which have already been downloaded). @@ -43,9 +44,7 @@ See `python shanoir_downloader_check.py --help` for more information. You might want to skip the anonymization process and the encryption process with the `--skip_anonymization` and `--skip_encryption` arguments respectively (or `-sa` and `-se`). -### `shanoir2bids.py` - -A `.json` configuration file must be provided to transform a Shanoir dataset into a BIDS dataset. +For `shanoir2bids.py`, a configuration file must be provided to transform a Shanoir dataset into a BIDS dataset. ``` -----------------------------[.json configuration file information]------------------------------- This file will tell the script what Shanoir datasets should be downloaded and how the data will be organised. @@ -56,33 +55,31 @@ The dictionary in the json file must have four keys : "data_to_bids": list of dict, each dictionary specifies datasets to download and BIDS format with the following keys : -> "datasetName": str, Shanoir name for the sequence to search -> "bidsDir" : str, BIDS subdirectory sequence name (eg : "anat", "func" or "dwi", ...) - -> "bidsName" : str, BIDS sequence name (eg: "t1w", "acq-b0_dir-AP", ...) + -> "bidsName" : str, BIDS sequence name (eg: "t1w-mprage", "t2-hr", "cusp66-ap-b0", ...) ``` -Please refer to the [BIDS starter kit](https://bids-standard.github.io/bids-starter-kit/folders_and_files/files.html) -for exhaustive templates of filenames. A BIDS compatible example is provided in the file `s2b_example_config.json`. + +An example is provided in the file `s2b_example_config.json`. To download longitudinal data, a key `session` and a new entry `bidsSession` in `data_to_bids` dictionaries should be defined in the JSON configuration files. Of note, only one session can be downloaded at once. Then, the key `session` is just a string, not a list as for subjects. -### Download Examples -#### Raw download +### Example usage + To download datasets, verify the content of them, anonymize them and / or encrypt them you can use a command like: `python shanoir_downloader_check.py -u username -d shanoir.irisa.fr -ids path/to/datasets_to_download.csv -of path/to/output/folder/ -se -lf path/to/downloads.log` The `example_input_check.csv` file in this repository is an example input file (the format of the `datasets_to_download.csv` file should be the same). -#### Solr search download You can also download datasets from a [SolR search](https://shanoir.irisa.fr/shanoir-ng/solr-search) as on the website: `python shanoir_downloader.py -u amasson -d shanoir.irisa.fr -of /data/amasson/test/shanoir_test4 --search_text "FLAIR" -p 1 -s 2 ` where `--search_text` is the string you would use on [the SolR search page](https://shanoir.irisa.fr/shanoir-ng/solr-search) (for example `(subjectName:(CT* OR demo*) AND studyName:etude test) OR datasetName:*flair*`). More information on the info box of the SolR search page. -#### BIDS download -`python shanoir2bids.py -j s2b_example_config.json -of my_download_dir --outformat nifti` will download Shanoir datasets identified in the configuration file saves them as DICOM and convert them into a BIDS datalad dataset into `my_download_dir`. +`python shanoir2bids.py -c s2b_example_config.json -d my_download_dir` will download Shanoir files identified in the configuration file and saves them as a BIDS data structure into `my_download_dir` -## About Solr Search +### Search usage The `--search_text` and `--expert_mode` arguments work as on the [Shanoir search page](https://shanoir.irisa.fr/shanoir-ng/solr-search). diff --git a/environment.yml b/environment.yml index 2465de7..7a0d5fd 100644 --- a/environment.yml +++ b/environment.yml @@ -1,4 +1,4 @@ -name: test-install +name: working-env channels: - conda-forge - defaults From 6e4b1d3b0aa07c311c0fe81f32bee766629877f2 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Thu, 18 Jul 2024 16:28:41 +0200 Subject: [PATCH 69/86] [BF]: fix pip and conda installation syntax --- .github/workflows/conda-installation.yml | 5 +- environment.yml | 208 ----------------------- 2 files changed, 4 insertions(+), 209 deletions(-) delete mode 100644 environment.yml diff --git a/.github/workflows/conda-installation.yml b/.github/workflows/conda-installation.yml index d8becbc..198d557 100644 --- a/.github/workflows/conda-installation.yml +++ b/.github/workflows/conda-installation.yml @@ -16,9 +16,12 @@ jobs: uses: conda-incubator/setup-miniconda@v3 with: auto-update-conda: true - environment-file: environment.yml activate-environment: working-env python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install -e . + conda install -c conda-forge heudiconv git-annex=*alldep datalaad # Create virtual env under each python version # (optional and a bit redundant because we are already on a specific python) diff --git a/environment.yml b/environment.yml deleted file mode 100644 index 7a0d5fd..0000000 --- a/environment.yml +++ /dev/null @@ -1,208 +0,0 @@ -name: working-env -channels: - - conda-forge - - defaults -dependencies: - - _libgcc_mutex=0.1=conda_forge - - _openmp_mutex=4.5=2_gnu - - annexremote=1.6.5=py310hff52083_0 - - atk-1.0=2.36.0=h516909a_2 - - backports=1.0=pyhd8ed1ab_3 - - backports.tarfile=1.0.0=pyhd8ed1ab_1 - - boto3=1.34.144=pyhd8ed1ab_0 - - botocore=1.34.144=pyge310_1234567_0 - - brotli=1.1.0=hd590300_1 - - brotli-bin=1.1.0=hd590300_1 - - brotli-python=1.0.9=py310hd8f1fbe_7 - - bz2file=0.98=py_0 - - bzip2=1.0.8=h5eee18b_6 - - c-ares=1.32.2=h4bc722e_0 - - ca-certificates=2024.7.4=hbcca054_0 - - cached-property=1.5.2=hd8ed1ab_1 - - cached_property=1.5.2=pyha770c72_1 - - cairo=1.16.0=hb05425b_5 - - cffi=1.16.0=py310h2fee648_0 - - chardet=5.2.0=py310hff52083_1 - - ci-info=0.3.0=pyhd8ed1ab_0 - - click=8.1.7=unix_pyh707e725_0 - - colorama=0.4.6=pyhd8ed1ab_0 - - cryptography=42.0.8=py310hb1bd9d3_0 - - curl=8.7.1=hdbd6064_0 - - cycler=0.12.1=pyhd8ed1ab_0 - - datalad=1.1.1=py310hff52083_0 - - dbus=1.13.6=he372182_0 - - dcm2niix=1.0.20211006=h4bd325d_0 - - dcmstack=0.8.0=pyhd8ed1ab_3 - - distro=1.9.0=pyhd8ed1ab_0 - - etelemetry=0.3.1=pyhd8ed1ab_0 - - exifread=3.0.0=pyhd8ed1ab_0 - - expat=2.6.2=h59595ed_0 - - fasteners=0.17.3=pyhd8ed1ab_0 - - filelock=3.15.4=pyhd8ed1ab_0 - - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 - - font-ttf-inconsolata=3.000=h77eed37_0 - - font-ttf-source-code-pro=2.038=h77eed37_0 - - font-ttf-ubuntu=0.83=h77eed37_2 - - fontconfig=2.14.1=h4c34cd2_2 - - fonts-conda-ecosystem=1=0 - - fonts-conda-forge=1=0 - - fonttools=4.53.1=py310h5b4e0ec_0 - - freetype=2.10.4=h0708190_1 - - fribidi=1.0.10=h36c2ea0_0 - - gdk-pixbuf=2.42.10=h5eee18b_0 - - gettext=0.21.1=h27087fc_0 - - git=2.33.0=pl5321hf874766_1 - - git-annex=10.20230626=nodep_h1234567_1 - - glib=2.69.1=he621ea3_2 - - gobject-introspection=1.72.0=py310hbb6d50b_2 - - graphite2=1.3.14=h295c915_1 - - graphviz=2.50.0=h3cd0ef9_0 - - gtk2=2.24.33=h73c1081_2 - - gts=0.7.6=h08bb679_0 - - h2=4.1.0=pyhd8ed1ab_0 - - h5py=3.7.0=nompi_py310h06dffec_100 - - harfbuzz=4.3.0=hf52aaf7_2 - - hdf5=1.12.1=h2b7332f_3 - - heudiconv=1.1.6=pyhd8ed1ab_0 - - hpack=4.0.0=pyh9f0ad1d_0 - - humanize=4.10.0=pyhd8ed1ab_0 - - hyperframe=6.0.1=pyhd8ed1ab_0 - - icu=73.1=h6a678d5_0 - - importlib_metadata=8.0.0=hd8ed1ab_0 - - importlib_resources=6.4.0=pyhd8ed1ab_0 - - iso8601=2.1.0=pyhd8ed1ab_0 - - isodate=0.6.1=pyhd8ed1ab_0 - - jaraco.classes=3.4.0=pyhd8ed1ab_1 - - jaraco.context=5.3.0=pyhd8ed1ab_1 - - jaraco.functools=4.0.0=pyhd8ed1ab_0 - - jeepney=0.8.0=pyhd8ed1ab_0 - - jmespath=1.0.1=pyhd8ed1ab_0 - - jpeg=9e=h0b41bf4_3 - - keyring=25.2.1=pyha804496_0 - - keyrings.alt=5.0.1=pyhd8ed1ab_1 - - kiwisolver=1.4.2=py310hbf28c38_1 - - krb5=1.20.1=h143b758_1 - - lcms2=2.15=hfd0df8a_0 - - ld_impl_linux-64=2.38=h1181459_1 - - lerc=3.0=h9c3ff4c_0 - - libblas=3.9.0=20_linux64_openblas - - libbrotlicommon=1.1.0=hd590300_1 - - libbrotlidec=1.1.0=hd590300_1 - - libbrotlienc=1.1.0=hd590300_1 - - libcblas=3.9.0=20_linux64_openblas - - libcurl=8.7.1=h251f7ec_0 - - libdeflate=1.17=h0b41bf4_0 - - libedit=3.1.20230828=h5eee18b_0 - - libev=4.33=hd590300_2 - - libexpat=2.6.2=h59595ed_0 - - libffi=3.4.4=h6a678d5_1 - - libgcc-ng=14.1.0=h77fa898_0 - - libgd=2.3.3=h695aa2c_1 - - libgfortran-ng=14.1.0=h69a702a_0 - - libgfortran5=14.1.0=hc5f4f2c_0 - - libgomp=14.1.0=h77fa898_0 - - libiconv=1.17=hd590300_2 - - liblapack=3.9.0=20_linux64_openblas - - libnghttp2=1.57.0=h2d74bed_0 - - libopenblas=0.3.25=pthreads_h413a1c8_0 - - libpng=1.6.39=h5eee18b_0 - - librsvg=2.54.4=h36cc946_3 - - libssh2=1.10.0=ha35d2d1_2 - - libstdcxx-ng=11.2.0=h1234567_1 - - libtiff=4.5.1=h6a678d5_0 - - libtool=2.4.7=h27087fc_0 - - libuuid=1.41.5=h5eee18b_0 - - libwebp-base=1.4.0=hd590300_0 - - libxcb=1.16=hd590300_0 - - libxcrypt=4.4.36=hd590300_1 - - libxml2=2.10.4=hfdd30dd_2 - - libxslt=1.1.37=h873f0b0_0 - - looseversion=1.3.0=pyhd8ed1ab_0 - - lxml=4.9.1=py310h5764c6d_0 - - lz4-c=1.9.4=h6a678d5_1 - - matplotlib-base=3.5.2=py310h5701ce4_0 - - more-itertools=10.3.0=pyhd8ed1ab_0 - - msgpack-python=1.0.3=py310hbf28c38_1 - - munkres=1.1.4=pyh9f0ad1d_0 - - mutagen=1.47.0=pyhd8ed1ab_0 - - ncurses=6.4=h6a678d5_0 - - networkx=3.2=pyhd8ed1ab_0 - - nipype=1.8.6=py310hff52083_0 - - numpy=1.22.3=py310h4ef5377_2 - - openjpeg=2.4.0=h9ca470c_2 - - openssl=3.3.1=h4bc722e_2 - - p7zip=16.02=h9c3ff4c_1001 - - packaging=24.1=pyhd8ed1ab_0 - - pango=1.50.7=h05da053_0 - - pathlib=1.0.1=py310hff52083_7 - - patool=2.3.0=pyhd8ed1ab_0 - - pcre=8.45=h9c3ff4c_0 - - pcre2=10.37=h032f7d1_0 - - perl=5.32.1=7_hd590300_perl5 - - pip=24.0=py310h06a4308_0 - - pixman=0.40.0=h36c2ea0_0 - - platformdirs=4.2.2=pyhd8ed1ab_0 - - prov=2.0.0=pyhd3deb0d_0 - - psutil=6.0.0=py310hc51659f_0 - - pthread-stubs=0.4=h36c2ea0_1001 - - pycparser=2.22=pyhd8ed1ab_0 - - pydot=3.0.1=py310hff52083_0 - - pyparsing=3.1.2=pyhd8ed1ab_0 - - pyperclip=1.8.2=pyhd8ed1ab_2 - - pysocks=1.7.1=pyha2e5f31_6 - - python=3.10.14=h955ad1f_1 - - python-gitlab=4.8.0=pyhd8ed1ab_0 - - python_abi=3.10=2_cp310 - - rdflib=7.0.0=pyhd8ed1ab_0 - - readline=8.2=h5eee18b_0 - - requests-ftp=0.3.1=py_1 - - requests-toolbelt=1.0.0=pyhd8ed1ab_0 - - s3transfer=0.10.2=pyhd8ed1ab_0 - - secretstorage=3.3.3=py310hff52083_2 - - setuptools=69.5.1=py310h06a4308_0 - - simplejson=3.19.2=py310h2372a71_0 - - six=1.16.0=pyh6c4a22f_0 - - sqlite=3.45.3=h5eee18b_0 - - tk=8.6.14=h39e8969_0 - - traits=6.3.2=py310h5764c6d_1 - - typing_extensions=4.12.2=pyha770c72_0 - - tzdata=2024a=h04d1e81_0 - - unicodedata2=15.1.0=py310h2372a71_0 - - wheel=0.43.0=py310h06a4308_0 - - whoosh=2.7.4=py310hff52083_8 - - xorg-libxau=1.0.11=hd590300_0 - - xorg-libxdmcp=1.1.3=h7f98852_0 - - xvfbwrapper=0.2.9=pyhd8ed1ab_1005 - - xz=5.4.6=h5eee18b_1 - - zlib=1.2.13=h5eee18b_1 - - zstandard=0.22.0=py310h1275a96_0 - - zstd=1.5.5=hc292b87_2 - - pip: - - certifi==2021.10.8 - - charset-normalizer==2.0.8 - - dicom2nifti==2.3.0 - - dicomanonymizer==0.0.1 - - idna==3.3 - - importlib-metadata==4.8.2 - - multivolumefile==0.2.3 - - nibabel==5.2.1 - - pandas==1.4.2 - - pillow==9.0.0 - - py7zr==0.17.0 - - pybcj==0.5.0 - - pycryptodomex==3.12.0 - - pydicom==2.2.2 - - pyppmd==0.17.3 - - python-dateutil==2.8.2 - - python-dotenv==0.19.2 - - pytz==2021.3 - - pyzstd==0.15.0 - - requests==2.26.0 - - scipy==1.11.4 - - simpleitk==2.1.1.2 - - texttable==1.6.4 - - tqdm==4.62.3 - - typing-extensions==4.0.1 - - urllib3==1.26.7 - - zipp==3.6.0 -prefix: /home/alpron/softs/miniconda3/envs/test From e1236850cf7af7973c79b28b6a2f3dfd015a1b79 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Thu, 18 Jul 2024 16:37:50 +0200 Subject: [PATCH 70/86] [BF]: fix pip and conda installation syntax --- .github/workflows/conda-installation.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/conda-installation.yml b/.github/workflows/conda-installation.yml index 198d557..28bb172 100644 --- a/.github/workflows/conda-installation.yml +++ b/.github/workflows/conda-installation.yml @@ -18,9 +18,11 @@ jobs: auto-update-conda: true activate-environment: working-env python-version: ${{ matrix.python-version }} + auto-activate-base: false - name: Install dependencies + shell: bash -el {0} run: | - python -m pip install -e . + python -m pip install . conda install -c conda-forge heudiconv git-annex=*alldep datalaad # Create virtual env under each python version From 3017c115ec6762379d052e6e7306375fcd70a1bf Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Thu, 18 Jul 2024 16:41:13 +0200 Subject: [PATCH 71/86] [BF]: fix conda syntax for build --- .github/workflows/conda-installation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/conda-installation.yml b/.github/workflows/conda-installation.yml index 28bb172..c87cba9 100644 --- a/.github/workflows/conda-installation.yml +++ b/.github/workflows/conda-installation.yml @@ -23,7 +23,7 @@ jobs: shell: bash -el {0} run: | python -m pip install . - conda install -c conda-forge heudiconv git-annex=*alldep datalaad + conda install -c conda-forge heudiconv git-annex=*=alldep* datalad # Create virtual env under each python version # (optional and a bit redundant because we are already on a specific python) From 36d94890f1cf64b08cf29d660b6334172ec0a908 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Thu, 18 Jul 2024 16:50:38 +0200 Subject: [PATCH 72/86] [BF]: updated installation instructions --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e0c44ab..28fb461 100644 --- a/README.md +++ b/README.md @@ -18,9 +18,13 @@ Optionally, rename the `.env.example` to `.env` and set the variables (`shanoir_ ### Installation with conda > [!IMPORTANT] -> This installation is required if shanoir2bids.py is used +> This installation method is required if shanoir2bids.py is used -After installing a (mini)conda distribution type in a terminal `conda create -f environment.yml` then activate the environment using `conda activate working-env` where `working-env` is the default name of the created environment +In an active conda virtual environment type +``` +pip install . +conda install -c conda-forge heudiconv git-annex=*=alldep* datalad +``` ## Usage There are three scripts to download datasets: From 303f6954c1f83abdc8eae6377aa54f14eab55047 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Fri, 19 Jul 2024 14:26:43 +0200 Subject: [PATCH 73/86] [BF]: updated conda installation and instructions so that pip packages are conda dependencies --- .github/workflows/conda-installation.yml | 4 +++- README.md | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/conda-installation.yml b/.github/workflows/conda-installation.yml index c87cba9..c7d1079 100644 --- a/.github/workflows/conda-installation.yml +++ b/.github/workflows/conda-installation.yml @@ -22,8 +22,10 @@ jobs: - name: Install dependencies shell: bash -el {0} run: | + conda config --set pip_interop_enabled True python -m pip install . - conda install -c conda-forge heudiconv git-annex=*=alldep* datalad + conda update --all + conda install -c conda-forge heudiconv bids-validator dcm2niix git-annex=*=alldep* datalad # Create virtual env under each python version # (optional and a bit redundant because we are already on a specific python) diff --git a/README.md b/README.md index 28fb461..eaeb567 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,13 @@ Optionally, rename the `.env.example` to `.env` and set the variables (`shanoir_ > This installation method is required if shanoir2bids.py is used In an active conda virtual environment type -``` +```bash +#use pip packages as dependencies +conda config --set pip_interop_enabled True pip install . +#replace pip packages with conda packages when equivalent +conda update --all +# install missing conda packages (far simpler than using pip) conda install -c conda-forge heudiconv git-annex=*=alldep* datalad ``` ## Usage From 20faca1fcbd6d186447169c736f7361483604e6a Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Thu, 28 Mar 2024 15:42:47 +0100 Subject: [PATCH 74/86] + --- s2b_example_config.json | 18 +- shanoir2bids_heudiconv.py | 542 +++++++++++++++++--------------------- 2 files changed, 241 insertions(+), 319 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 7286d24..f2a0163 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -9,23 +9,11 @@ {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b3000_dir-AP_dwi"}, {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-PA_dwi"} ], - "dcm2niix":"/home/qduche/Software/dcm2niix_lnx/dcm2niix", - "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", - "dcm2niix_options": { - "bids_format": true, - "anon_bids": true, - "compress": "y", - "compression": 5, - "crop": false, - "has_private": false, - "ignore_deriv": false, - "single_file": false, - "verbose": false - }, + "dcm2niix":"~/softs/miniconda3/bin/dcm2niix", + "dcm2niix_options": {"verbose": false,"compress": "y", "anon_bids": true}, "find_and_replace_subject": [ {"find":"VS_Aneravimm_", "replace": "VS"}, {"find":"Vs_Aneravimm_", "replace": "VS"} ] -} - +} \ No newline at end of file diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index 7f5ce66..f0d224d 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -4,7 +4,8 @@ The script is made to run for every project given some information provided by the user into a ".json" configuration file. More details regarding the configuration file in the Readme.md""" # Script to download and BIDS-like organize data on Shanoir using "shanoir_downloader.py" developed by Arthur Masson - +# @Author: Malo Gaubert , Quentin Duché +# @Date: 24 Juin 2022 import os from os.path import join as opj, splitext as ops, exists as ope, dirname as opd @@ -26,7 +27,6 @@ # import loggger used in heudiconv workflow from heudiconv.main import lgr -import bids_validator # Load environment variables @@ -95,14 +95,6 @@ def banner_msg(msg): If you want to do so, add key "{key}" to you Shanoir2BIDS configuration file :""" -def create_tmp_directory(path_temporary_directory): - tmp_dir = Path(path_temporary_directory) - if tmp_dir.exists(): - shutil.rmtree(tmp_dir) - tmp_dir.mkdir(parents=True) - pass - - def check_date_format(date_to_format): # TRUE FORMAT should be: date_format = 'Y-m-dTH:M:SZ' try: @@ -180,10 +172,8 @@ def read_json_config_file(json_file): ) -def generate_bids_heuristic_file( - shanoir2bids_dict, - path_heuristic_file, - output_type='("dicom","nii.gz")', +def generate_heuristic_file( + shanoir2bids_dict: object, path_heuristic_file: object, output_type ) -> None: """Generate heudiconv heuristic.py file from shanoir2bids mapping dict Parameters @@ -191,9 +181,9 @@ def generate_bids_heuristic_file( shanoir2bids_dict : path_heuristic_file : path of the python heuristic file (.py) """ - if output_type == "dicom": + if output_type == 'dicom': outtype = '("dicom",)' - elif output_type == "nifti": + elif output_type == 'nifti': outtype = '("nii.gz",)' else: outtype = '("dicom","nii.gz")' @@ -243,10 +233,10 @@ def infotodict(seqinfo): with open(path_heuristic_file, "w", encoding="utf-8") as file: file.write(heuristic) + file.close() pass - class DownloadShanoirDatasetToBIDS: """ class that handles the downloading of shanoir data set and the reformatting as a BIDS data structure @@ -263,10 +253,10 @@ def __init__(self): self.shanoir_username = None # Shanoir username self.shanoir_study_id = None # Shanoir study ID self.shanoir_session_id = None # Shanoir study ID - self.shanoir_file_type = SHANOIR_FILE_TYPE_DICOM # Download File Type (DICOM) - self.output_file_type = ( - DEFAULT_SHANOIR_FILE_TYPE # Default Export File Type (NIFTI) + self.shanoir_file_type = ( + DEFAULT_SHANOIR_FILE_TYPE # Default download type (nifti/dicom) ) + self.output_file_type = DEFAULT_SHANOIR_FILE_TYPE self.json_config_file = None self.list_fars = [] # List of substrings to edit in subjects names self.dl_dir = None # download directory, where data will be stored @@ -274,16 +264,14 @@ def __init__(self): self.n_seq = 0 # Number of sequences in the shanoir2bids_dict self.log_fn = None self.dcm2niix_path = None # Path to the dcm2niix the user wants to use - self.actual_dcm2niix_path = shutil.which("dcm2niix") self.dcm2niix_opts = None # Options to add to the dcm2niix call self.date_from = None self.date_to = None self.longitudinal = False self.to_automri_format = ( - False # Special filenames for automri (No longer used ! --> BIDS format) + False # Special filenames for automri (close to BIDS format) ) self.add_sns = False # Add series number suffix to filename - self.debug_mode = False # No debug mode by default def set_json_config_file(self, json_file): """ @@ -313,11 +301,17 @@ def set_json_config_file(self, json_file): self.set_date_from(date_from=date_from) self.set_date_to(date_to=date_to) - def set_output_file_type(self, outfile_type): - if outfile_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, "both"]: - self.output_file_type = outfile_type + def set_shanoir_file_type(self, shanoir_file_type): + if shanoir_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI]: + self.shanoir_file_type = shanoir_file_type + else: + sys.exit("Unknown shanoir file type {}".format(shanoir_file_type)) + + def set_output_file_type(self, output_file_type): + if output_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, 'both']: + self.shanoir_file_type = output_file_type else: - sys.exit("Unknown output file type {}".format(outfile_type)) + sys.exit("Unknown shanoir file type {}".format(output_file_type)) def set_shanoir_study_id(self, study_id): self.shanoir_study_id = study_id @@ -390,84 +384,22 @@ def set_log_filename(self): def toggle_longitudinal_version(self): self.longitudinal = True - def is_correct_dcm2niix(self): - current_version = Path(self.actual_dcm2niix_path) - config_version = Path(self.dcm2niix_path) - if current_version is not None and config_version is not None: - return config_version.samefile(current_version) - else: - return False + def switch_to_automri_format(self): + self.to_automri_format = True + + def add_series_number_suffix(self): + self.add_sns = True def configure_parser(self): """ Configure the parser and the configuration of the shanoir_downloader """ self.parser = shanoir_downloader.create_arg_parser() - shanoir_downloader.add_username_argument(self.parser) - shanoir_downloader.add_domain_argument(self.parser) - self.parser.add_argument( - "-f", - "--format", - default="dicom", - choices=["dicom"], - help="The format to download.", - ) - shanoir_downloader.add_output_folder_argument(self.parser) + shanoir_downloader.add_common_arguments(self.parser) shanoir_downloader.add_configuration_arguments(self.parser) shanoir_downloader.add_search_arguments(self.parser) shanoir_downloader.add_ids_arguments(self.parser) - def is_mapping_bids(self): - """Check BIDS compliance of filenames/path used in the configuration file""" - validator = bids_validator.BIDSValidator() - - subjects = self.shanoir_subjects - list_find_and_replace = self.list_fars - if list_find_and_replace: - # normalise subjects name - normalised_subjects = [] - for subject in subjects: - for i, far in enumerate(list_find_and_replace): - if i == 0: - normalised_subject = subject - normalised_subject = normalised_subject.replace(far["find"], far["replace"]) - normalised_subjects.append(normalised_subject) - else: - normalised_subjects = subjects - - sessions = self.shanoir_session_id - extension = '.nii.gz' - - if sessions == '*': - paths = ( - "/" + "sub-" + subject + '/' + - map["bidsDir"] + '/' + - "sub-" + subject + '_' + - map["bidsName"] + extension - - for subject in normalised_subjects - for map in self.shanoir2bids_dict - ) - else: - paths = ( - "/" + "sub-" + subject + '/' + - "ses-" + session + '/' + - map["bidsDir"] + '/' + - "sub-" + subject + '_' + "ses-" + session + '_' + - map["bidsName"] + extension - - for session in sessions - for subject in normalised_subjects - for map in self.shanoir2bids_dict - ) - - bids_errors = [p for p in paths if not validator.is_bids(p)] - - if not bids_errors: - return True, bids_errors - else: - return False, bids_errors - def download_subject(self, subject_to_search): """ For a single subject @@ -483,203 +415,202 @@ def download_subject(self, subject_to_search): # Real Shanoir2Bids mapping (handle case when solr search term are included) bids_mapping = [] + # temporary directory containing dowloaded DICOM.zip files + with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_dicom: + with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_archive: + print(tmp_archive) + # Loop on each sequence defined in the dictionary + for seq in range(self.n_seq): + # Isolate elements that are called many times + shanoir_seq_name = self.shanoir2bids_dict[seq][ + K_DS_NAME + ] # Shanoir sequence name (OLD) + bids_seq_subdir = self.shanoir2bids_dict[seq][ + K_BIDS_DIR + ] # Sequence BIDS subdirectory name (NEW) + bids_seq_name = self.shanoir2bids_dict[seq][ + K_BIDS_NAME + ] # Sequence BIDS nickname (NEW) + if self.longitudinal: + bids_seq_session = self.shanoir2bids_dict[seq][ + K_BIDS_SES + ] # Sequence BIDS nickname (NEW) + + # Print message concerning the sequence that is being downloaded + print( + "\t-", + bids_seq_name, + subject_to_search, + "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", + ) - # Manual temporary directories containing dowloaded DICOM.zip and extracted files - # (temporary directories that can be kept are not supported by pythn <3.1 - tmp_dicom = Path(self.dl_dir).joinpath("tmp_dicoms", subject_to_search) - tmp_archive = Path(self.dl_dir).joinpath( - "tmp_archived_dicoms", subject_to_search - ) - create_tmp_directory(tmp_archive) - create_tmp_directory(tmp_dicom) - - # Loop on each sequence defined in the dictionary - for seq in range(self.n_seq): - # Isolate elements that are called many times - shanoir_seq_name = self.shanoir2bids_dict[seq][ - K_DS_NAME - ] # Shanoir sequence name (OLD) - bids_seq_subdir = self.shanoir2bids_dict[seq][ - K_BIDS_DIR - ] # Sequence BIDS subdirectory name (NEW) - bids_seq_name = self.shanoir2bids_dict[seq][ - K_BIDS_NAME - ] # Sequence BIDS nickname (NEW) - if self.longitudinal: - bids_seq_session = self.shanoir2bids_dict[seq][ - K_BIDS_SES - ] # Sequence BIDS nickname (NEW) - - # Print message concerning the sequence that is being downloaded - print( - "\t-", - bids_seq_name, - subject_to_search, - "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", - ) - - # Initialize the parser - search_txt = ( - "studyName:" - + self.shanoir_study_id.replace(" ", "?") - + " AND datasetName:" - + shanoir_seq_name.replace(" ", "?") - + " AND subjectName:" - + subject_to_search.replace(" ", "?") - + " AND examinationComment:" - + self.shanoir_session_id.replace(" ", "*") - + " AND examinationDate:[" - + self.date_from - + " TO " - + self.date_to - + "]" - ) - - args = self.parser.parse_args( - [ - "-u", - self.shanoir_username, - "-d", - self.shanoir_domaine, - "-of", - str(tmp_archive), - "-em", - "-st", - search_txt, - "-s", - "200", - "-f", - self.shanoir_file_type, - "-so", - "id,ASC", - "-t", - "500", - ] - ) # Increase time out for heavy files - - config = shanoir_downloader.initialize(args) - response = shanoir_downloader.solr_search(config, args) - - # From response, process the data - # Print the number of items found and a list of these items - if response.status_code == 200: - # Invoke shanoir_downloader to download all the data - shanoir_downloader.download_search_results(config, args, response) - - if len(response.json()["content"]) == 0: - warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns -a result on the website. -Search Text : "{}" \n""".format( - search_txt + # Initialize the parser + search_txt = ( + "studyName:" + + self.shanoir_study_id.replace(" ", "?") + + " AND datasetName:" + + shanoir_seq_name.replace(" ", "?") + + " AND subjectName:" + + subject_to_search.replace(" ", "?") + + " AND examinationComment:" + + self.shanoir_session_id.replace(" ", "*") + + " AND examinationDate:[" + + self.date_from + + " TO " + + self.date_to + + "]" ) - print(warn_msg) - fp.write(warn_msg) - else: - for item in response.json()["content"]: - # Define subject_id - su_id = item["subjectName"] - # If the user has defined a list of edits to subject names... then do the find and replace - for far in self.list_fars: - su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) - # ID of the subject (sub-*) - subject_id = su_id - - # correct BIDS mapping of the searched dataset - bids_seq_mapping = { - "datasetName": item["datasetName"], - "bidsDir": bids_seq_subdir, - "bidsName": bids_seq_name, - "bids_subject_id": subject_id, - } - - if self.longitudinal: - bids_seq_mapping["bids_session_id"] = bids_seq_session - else: - bids_seq_session = None - bids_seq_mapping["bids_session_id"] = bids_seq_session + args = self.parser.parse_args( + [ + "-u", + self.shanoir_username, + "-d", + self.shanoir_domaine, + "-of", + tmp_archive, + "-em", + "-st", + search_txt, + "-s", + "200", + "-f", + self.shanoir_file_type, + "-so", + "id,ASC", + "-t", + "500", + ] + ) # Increase time out for heavy files - bids_mapping.append(bids_seq_mapping) + config = shanoir_downloader.initialize(args) + response = shanoir_downloader.solr_search(config, args) - # Write the information on the data in the log file - fp.write("- datasetId = " + str(item["datasetId"]) + "\n") - fp.write(" -- studyName: " + item["studyName"] + "\n") - fp.write(" -- subjectName: " + item["subjectName"] + "\n") - fp.write(" -- session: " + item["examinationComment"] + "\n") - fp.write(" -- datasetName: " + item["datasetName"] + "\n") - fp.write( - " -- examinationDate: " + item["examinationDate"] + "\n" + # From response, process the data + # Print the number of items found and a list of these items + if response.status_code == 200: + # Invoke shanoir_downloader to download all the data + shanoir_downloader.download_search_results( + config, args, response ) - fp.write(" >> Downloading archive OK\n") - - # Extract the downloaded archive - dl_archive = glob(opj(tmp_archive, "*" + item["id"] + "*.zip"))[ - 0 - ] - with zipfile.ZipFile(dl_archive, "r") as zip_ref: - extraction_dir = opj(tmp_dicom, item["id"]) - zip_ref.extractall(extraction_dir) + if len(response.json()["content"]) == 0: + warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns + a result on the website. + Search Text : "{}" \n""".format( + search_txt + ) + print(warn_msg) + fp.write(warn_msg) + else: + for item in response.json()["content"]: + # Define subject_id + su_id = item["subjectName"] + # If the user has defined a list of edits to subject names... then do the find and replace + for far in self.list_fars: + su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) + + # ID of the subject (sub-*) + subject_id = su_id + # correct BIDS mapping of the searched dataset + bids_seq_mapping = { + "datasetName": item["datasetName"], + "bidsDir": bids_seq_subdir, + "bidsName": bids_seq_name, + "bids_subject_id": subject_id, + } + + if self.longitudinal: + bids_seq_mapping[ + "bids_session_id" + ] = bids_seq_session + else: + bids_seq_mapping["bids_session_id"] = None + + bids_mapping.append(bids_seq_mapping) + + # Write the information on the data in the log file + fp.write( + "- datasetId = " + str(item["datasetId"]) + "\n" + ) + fp.write(" -- studyName: " + item["studyName"] + "\n") + fp.write( + " -- subjectName: " + item["subjectName"] + "\n" + ) + fp.write( + " -- session: " + item["examinationComment"] + "\n" + ) + fp.write( + " -- datasetName: " + item["datasetName"] + "\n" + ) + fp.write( + " -- examinationDate: " + + item["examinationDate"] + + "\n" + ) + fp.write(" >> Downloading archive OK\n") + + # Extract the downloaded archive + dl_archive = glob( + opj(tmp_archive, "*" + item["id"] + "*.zip") + )[0] + with zipfile.ZipFile(dl_archive, "r") as zip_ref: + extraction_dir = opj(tmp_dicom, item["id"]) + zip_ref.extractall(extraction_dir) + + fp.write( + " >> Extraction of all files from archive '" + + dl_archive + + " into " + + extraction_dir + + "\n" + ) + + elif response.status_code == 204: + banner_msg("ERROR : No file found!") + fp.write(" >> ERROR : No file found!\n") + else: + banner_msg( + "ERROR : Returned by the request: status of the response = " + + response.status_code + ) fp.write( - " >> Extraction of all files from archive '" - + dl_archive - + " into " - + extraction_dir + " >> ERROR : Returned by the request: status of the response = " + + str(response.status_code) + "\n" ) - elif response.status_code == 204: - banner_msg("ERROR : No file found!") - fp.write(" >> ERROR : No file found!\n") - else: - banner_msg( - "ERROR : Returned by the request: status of the response = " - + response.status_code - ) - fp.write( - " >> ERROR : Returned by the request: status of the response = " - + str(response.status_code) - + "\n" - ) - - # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options - with tempfile.NamedTemporaryFile( - mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" - ) as heuristic_file: - # Generate Heudiconv heuristic file from configuration.json mapping - generate_bids_heuristic_file( - bids_mapping, heuristic_file.name, output_type=self.output_file_type - ) + # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options with tempfile.NamedTemporaryFile( - mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" - ) as dcm2niix_config_file: - self.export_dcm2niix_config_options(dcm2niix_config_file.name) - workflow_params = { - "files": glob(opj(tmp_dicom, "*", "*.dcm"), recursive=True), - "outdir": opj(self.dl_dir, str(self.shanoir_study_id)), - "subjs": [subject_id], - "converter": "dcm2niix", - "heuristic": heuristic_file.name, - "bids_options": "--bids", - # "with_prov": True, - "dcmconfig": dcm2niix_config_file.name, - "datalad": True, - "minmeta": True, - "grouping": "all", # other options are too restrictive (tested on EMISEP) - "overwrite": True, - } - - if self.longitudinal: - workflow_params["session"] = bids_seq_session - - workflow(**workflow_params) - fp.close() - if not self.debug_mode: - shutil.rmtree(tmp_archive, ignore_errors=True) - shutil.rmtree(tmp_dicom, ignore_errors=True) - # beware of side effects - shutil.rmtree(tmp_archive.parent, ignore_errors=True) - shutil.rmtree(tmp_dicom.parent, ignore_errors=True) + mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" + ) as heuristic_file: + # Generate Heudiconv heuristic file from configuration.json mapping + generate_heuristic_file(bids_mapping, heuristic_file.name, output_type=self.output_file_type) + with tempfile.NamedTemporaryFile( + mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" + ) as dcm2niix_config_file: + self.export_dcm2niix_config_options(dcm2niix_config_file.name) + workflow_params = { + "files": glob(opj(tmp_dicom, "*", "*.dcm"), recursive=True), + "outdir": opj(self.dl_dir, str(self.shanoir_study_id)), + "subjs": [subject_id], + "converter": "dcm2niix", + "heuristic": heuristic_file.name, + "bids_options": "--bids", + # "with_prov": True, + "dcmconfig": dcm2niix_config_file.name, + "datalad": True, + "minmeta": True, + "grouping": "all", # other options are too restrictive (tested on EMISEP) + } + + if self.longitudinal: + workflow_params["session"] = bids_seq_session + + workflow(**workflow_params) + # TODO add nipype logging into shanoir log file ? + # TODO use provenance option ? currently not working properly + fp.close() def download(self): """ @@ -713,10 +644,16 @@ def main(): default="shanoir.irisa.fr", help="The shanoir domain to query.", ) - + # parser.add_argument( + # "-f", + # "--format", + # default="dicom", + # choices=["dicom"], + # help="The format to download.", + # ) parser.add_argument( "--outformat", - default="both", + default="nifti", choices=["nifti", "dicom", "both"], help="The format to download.", ) @@ -736,13 +673,17 @@ def main(): action="store_true", help="Toggle longitudinal approach.", ) - parser.add_argument( - "--debug", - required=False, - action="store_true", - help="Toggle debug mode (keep temporary directories)", - ) + # parser.add_argument( + # "-a", "--automri", action="store_true", help="Switch to automri file tree." + # ) + # parser.add_argument( + # "-A", + # "--add_sns", + # action="store_true", + # help="Add series number suffix (compatible with -a)", + # ) + # Parse arguments args = parser.parse_args() # Start configuring the DownloadShanoirDatasetToBids class instance @@ -752,28 +693,21 @@ def main(): stb.set_json_config_file( json_file=args.config_file ) # path to json configuration file - stb.set_output_file_type(args.outformat) + stb.set_output_file_type(output_file_type=args.outformat) stb.set_download_directory( dl_dir=args.output_folder ) # output folder (if None a default directory is created) - if args.debug: - stb.debug_mode = True - if args.longitudinal: stb.toggle_longitudinal_version() - - if not stb.is_correct_dcm2niix(): - print( - f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}" - ) - else: - if stb.is_mapping_bids()[0]: - stb.download() - else: - print( - f"Provided BIDS keys {stb.is_mapping_bids()[1]} are not BIDS compliant check syntax in provided configuration file {args.config_file}" - ) + # if args.automri: + # stb.switch_to_automri_format() + # if args.add_sns: + # if not args.automri: + # print("Warning : -A option is only compatible with -a option.") + # stb.add_series_number_suffix() + + stb.download() if __name__ == "__main__": From a67dd062ec4ce1656473a3e02fabc036bfafa02c Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 9 Apr 2024 10:53:14 +0200 Subject: [PATCH 75/86] clean parser options --- s2b_example_config.json | 2 +- shanoir2bids_heudiconv.py | 35 ++++++++++++++--------------------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index f2a0163..199f2c6 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -9,7 +9,7 @@ {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b3000_dir-AP_dwi"}, {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-PA_dwi"} ], - "dcm2niix":"~/softs/miniconda3/bin/dcm2niix", + "dcm2niix":"/home/alpron/softs/miniconda3/bin/dcm2niix", "dcm2niix_options": {"verbose": false,"compress": "y", "anon_bids": true}, "find_and_replace_subject": [ diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index f0d224d..c31fadc 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -173,7 +173,7 @@ def read_json_config_file(json_file): def generate_heuristic_file( - shanoir2bids_dict: object, path_heuristic_file: object, output_type + shanoir2bids_dict: object, path_heuristic_file: object, output_type='("dicom","nii.gz")' ) -> None: """Generate heudiconv heuristic.py file from shanoir2bids mapping dict Parameters @@ -233,7 +233,6 @@ def infotodict(seqinfo): with open(path_heuristic_file, "w", encoding="utf-8") as file: file.write(heuristic) - file.close() pass @@ -253,10 +252,8 @@ def __init__(self): self.shanoir_username = None # Shanoir username self.shanoir_study_id = None # Shanoir study ID self.shanoir_session_id = None # Shanoir study ID - self.shanoir_file_type = ( - DEFAULT_SHANOIR_FILE_TYPE # Default download type (nifti/dicom) - ) - self.output_file_type = DEFAULT_SHANOIR_FILE_TYPE + self.shanoir_file_type = SHANOIR_FILE_TYPE_DICOM # Download File Type (DICOM) + self.output_file_type = DEFAULT_SHANOIR_FILE_TYPE # Default Export File Type (NIFTI) self.json_config_file = None self.list_fars = [] # List of substrings to edit in subjects names self.dl_dir = None # download directory, where data will be stored @@ -301,17 +298,11 @@ def set_json_config_file(self, json_file): self.set_date_from(date_from=date_from) self.set_date_to(date_to=date_to) - def set_shanoir_file_type(self, shanoir_file_type): - if shanoir_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI]: - self.shanoir_file_type = shanoir_file_type - else: - sys.exit("Unknown shanoir file type {}".format(shanoir_file_type)) - - def set_output_file_type(self, output_file_type): - if output_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, 'both']: - self.shanoir_file_type = output_file_type + def set_output_file_type(self, outfile_type): + if outfile_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, 'both']: + self.output_file_type = outfile_type else: - sys.exit("Unknown shanoir file type {}".format(output_file_type)) + sys.exit("Unknown output file type {}".format(outfile_type)) def set_shanoir_study_id(self, study_id): self.shanoir_study_id = study_id @@ -395,7 +386,11 @@ def configure_parser(self): Configure the parser and the configuration of the shanoir_downloader """ self.parser = shanoir_downloader.create_arg_parser() - shanoir_downloader.add_common_arguments(self.parser) + shanoir_downloader.add_username_argument(self.parser) + shanoir_downloader.add_domain_argument(self.parser) + self.parser.add_argument('-f', '--format', default='dicom', choices=['dicom'], + help='The format to download.') + shanoir_downloader.add_output_folder_argument(self.parser) shanoir_downloader.add_configuration_arguments(self.parser) shanoir_downloader.add_search_arguments(self.parser) shanoir_downloader.add_ids_arguments(self.parser) @@ -608,8 +603,6 @@ def download_subject(self, subject_to_search): workflow_params["session"] = bids_seq_session workflow(**workflow_params) - # TODO add nipype logging into shanoir log file ? - # TODO use provenance option ? currently not working properly fp.close() def download(self): @@ -653,7 +646,7 @@ def main(): # ) parser.add_argument( "--outformat", - default="nifti", + default="both", choices=["nifti", "dicom", "both"], help="The format to download.", ) @@ -693,7 +686,7 @@ def main(): stb.set_json_config_file( json_file=args.config_file ) # path to json configuration file - stb.set_output_file_type(output_file_type=args.outformat) + stb.set_output_file_type(args.outformat) stb.set_download_directory( dl_dir=args.output_folder ) # output folder (if None a default directory is created) From e2485f3d0cb49f948cd644e98d4101bb4f5abc2a Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 22 Apr 2024 12:46:40 +0200 Subject: [PATCH 76/86] added main dcm2niix configuration options into nipype format + documentation link as comment entry --- s2b_example_config.json | 24 +- s2b_example_config_EMISEP_long.json | 363 ++++++---------------------- 2 files changed, 85 insertions(+), 302 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 199f2c6..5407eba 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -9,11 +9,21 @@ {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b3000_dir-AP_dwi"}, {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-PA_dwi"} ], - "dcm2niix":"/home/alpron/softs/miniconda3/bin/dcm2niix", - "dcm2niix_options": {"verbose": false,"compress": "y", "anon_bids": true}, - "find_and_replace_subject": - [ - {"find":"VS_Aneravimm_", "replace": "VS"}, - {"find":"Vs_Aneravimm_", "replace": "VS"} - ] + "dcm2niix": "/home/alpron/softs/miniconda3/bin/dcm2niix", + "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", + "dcm2niix_options": { + "bids_format": true, + "anon_bids": true, + "compress": "y", + "compression": 5, + "crop": false, + "has_private": false, + "ignore_deriv": false, + "single_file": false, + "verbose": false + }, + "find_and_replace_subject": [ + {"find": "VS_Aneravimm_", "replace": "VS"}, + {"find": "Vs_Aneravimm_", "replace": "VS"} + ] } \ No newline at end of file diff --git a/s2b_example_config_EMISEP_long.json b/s2b_example_config_EMISEP_long.json index e22a037..c7bfb4a 100644 --- a/s2b_example_config_EMISEP_long.json +++ b/s2b_example_config_EMISEP_long.json @@ -1,299 +1,70 @@ { - "study_name": "EMISEP", - "subjects": ["08-74"], - "session": "08-74 MO ENCEPHALE", - "data_to_bids": [ - { - "datasetName": "3D T1 MPRAGE", - "bidsDir": "brain", - "bidsName": "T1w", - "bidsSession": "M00" - }, - { - "datasetName": "*3d flair*", - "bidsDir": "brain", - "bidsName": "FLAIR", - "bidsSession": "M00" - }, - { - "datasetName": "*c1c3*", - "bidsDir": "spinalCord", - "bidsName": "t2ax_C1C3", - "bidsSession": "M00" - }, - { - "datasetName": "*c1-c3*", - "bidsDir": "spinalCord", - "bidsName": "t2ax_C1C3", - "bidsSession": "M00" - }, - { - "datasetName": "*c4c7*", - "bidsDir": "spinalCord", - "bidsName": "t2ax_C4C7", - "bidsSession": "M00" - }, - { - "datasetName": "*c4-c7*", - "bidsDir": "spinalCord", - "bidsName": "t2ax_C4C7", - "bidsSession": "M00" - }, - { - "datasetName": "t2_tse_rr_p2_sag", - "bidsDir": "spinalCord", - "bidsName": "t2Sag", - "bidsSession": "M00" - }, - { - "datasetName": "t2_tse_rr_p2_sag2.5 te 81", - "bidsDir": "spinalCord", - "bidsName": "t2Sag", - "bidsSession": "M00" - }, - { - "datasetName": "t2_tse_sag_384", - "bidsDir": "spinalCord", - "bidsName": "t2Sag", - "bidsSession": "M00" - }, - { - "datasetName": "t2_tse_sag_p2", - "bidsDir": "spinalCord", - "bidsName": "t2Sag", - "bidsSession": "M00" - }, - { - "datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260", - "bidsDir": "spinalCord", - "bidsName": "t2Sag", - "bidsSession": "M00" - }, - { - "datasetName": "SAG T2 CERV", - "bidsDir": "spinalCord", - "bidsName": "t2Sag", - "bidsSession": "M00" - }, - { - "datasetName": "SAG T2 DORSAL", - "bidsDir": "spinalCord", - "bidsName": "t2Sag", - "bidsSession": "M00" - }, - { - "datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp", - "bidsDir": "spinalCord", - "bidsName": "t2Sag-COMP", - "bidsSession": "M00" - }, - { - "datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp_sp", - "bidsDir": "spinalCord", - "bidsName": "t2Sag-COMP", - "bidsSession": "M00" - }, - { - "datasetName": "t2_tse_rr_p2_sag_comp_sp", - "bidsDir": "spinalCord", - "bidsName": "t2Sag-COMP", - "bidsSession": "M00" - }, - { - "datasetName": "t2_tse_sag_384_comp_sp", - "bidsDir": "spinalCord", - "bidsName": "t2Sag-COMP", - "bidsSession": "M00" - }, - { - "datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260_comp_ad", - "bidsDir": "spinalCord", - "bidsName": "t2Sag-COMP", - "bidsSession": "M00" - }, - { - "datasetName": "t2_tse_sag_p2_comp_ad", - "bidsDir": "spinalCord", - "bidsName": "t2Sag-COMP", - "bidsSession": "M00" - }, - { - "datasetName": "SAG T2 TSE MOELLE", - "bidsDir": "spinalCord", - "bidsName": "t2Sag-COMP", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_sag_p2_iso", - "bidsDir": "spinalCord", - "bidsName": "mt0", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_sag_p2_iso mt", - "bidsDir": "spinalCord", - "bidsName": "mt1", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_tra_p2_iso", - "bidsDir": "spinalCord", - "bidsName": "mt0", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_tra_p2_iso mt", - "bidsDir": "spinalCord", - "bidsName": "mt1", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_we_tra_p2_iso", - "bidsDir": "spinalCord", - "bidsName": "mt0", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_we_tra_p2_iso mt tr38", - "bidsDir": "spinalCord", - "bidsName": "mt1", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_we_tra_p2_iso mt tr38 sans sat", - "bidsDir": "spinalCord", - "bidsName": "mt1", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_we_tra_p2_iso mt_tr38 sans sat", - "bidsDir": "spinalCord", - "bidsName": "mt1", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_we_tra_p2_iso tr38", - "bidsDir": "spinalCord", - "bidsName": "mt0", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt", - "bidsDir": "spinalCord", - "bidsName": "mt1", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt sans sat", - "bidsDir": "spinalCord", - "bidsName": "mt1", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat", - "bidsDir": "spinalCord", - "bidsName": "mt0", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat_mt", - "bidsDir": "spinalCord", - "bidsName": "mt1", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38 sans sat", - "bidsDir": "spinalCord", - "bidsName": "mt1", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38_sans_sat", - "bidsDir": "spinalCord", - "bidsName": "mt1", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_we_tra_p2_iso_tr38 sans sat", - "bidsDir": "spinalCord", - "bidsName": "mt0", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_we_tra_p2_m0_iso tr38 sans sat", - "bidsDir": "spinalCord", - "bidsName": "mt0", - "bidsSession": "M00" - }, - { - "datasetName": "t1_fl3d_we_tra_p2_mt_iso tr38", - "bidsDir": "spinalCord", - "bidsName": "mt1", - "bidsSession": "M00" - }, - { - "datasetName": "3d_t1_mprage_sag", - "bidsDir": "spinalCord", - "bidsName": "t1Sag", - "bidsSession": "M00" - }, - { - "datasetName": "t1_mprage_sag_p2_iso", - "bidsDir": "spinalCord", - "bidsName": "t1Sag", - "bidsSession": "M00" - }, - { - "datasetName": "t1_mprage_sag_p2_iso cerv", - "bidsDir": "spinalCord", - "bidsName": "t1Sag", - "bidsSession": "M00" - }, - { - "datasetName": "t1_mprage_sag_p2_iso cerv_rr", - "bidsDir": "spinalCord", - "bidsName": "t1Sag", - "bidsSession": "M00" - }, - { - "datasetName": "t1_mprage_sag_p2_iso ofsep", - "bidsDir": "spinalCord", - "bidsName": "t1Sag", - "bidsSession": "M00" - }, - { - "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_axial", - "bidsDir": "spinalCord", - "bidsName": "t1Sag", - "bidsSession": "M00" - }, - { - "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_cor", - "bidsDir": "spinalCord", - "bidsName": "t1Sag", - "bidsSession": "M00" - }, - { - "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_tra", - "bidsDir": "spinalCord", - "bidsName": "t1Sag", - "bidsSession": "M00" - }, - { - "datasetName": "t1_mprage_sag_p2_iso ofsep_rr", - "bidsDir": "spinalCord", - "bidsName": "t1Sag", - "bidsSession": "M00" - }, - { - "datasetName": "T1 MPRAGE CERV", - "bidsDir": "spinalCord", - "bidsName": "t1Sag", - "bidsSession": "M00" - } - ], - "dcm2niix": "~/softs/miniconda3/bin/dcm2niix", - "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", + "study_name": "EMISEP", + "subjects": ["08-74"], + "session": "08-74 MO ENCEPHALE", + "data_to_bids": + [ + {"datasetName": "3D T1 MPRAGE", "bidsDir": "brain", "bidsName": "T1w", "bidsSession": "M00"}, + {"datasetName": "*3d flair*", "bidsDir": "brain", "bidsName": "FLAIR", "bidsSession": "M00"}, + {"datasetName": "*c1c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", "bidsSession": "M00"}, + {"datasetName": "*c1-c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", "bidsSession": "M00"}, + + {"datasetName": "*c4c7*", "bidsDir": "spinalCord", "bidsName": "t2ax_C4C7", "bidsSession": "M00"}, + {"datasetName": "*c4-c7*", "bidsDir": "spinalCord", "bidsName": "t2ax_C4C7", "bidsSession": "M00"}, + + + {"datasetName": "t2_tse_rr_p2_sag", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, + {"datasetName": "t2_tse_rr_p2_sag2.5 te 81", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, + {"datasetName": "t2_tse_sag_384", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, + {"datasetName": "t2_tse_sag_p2", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, + {"datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, + {"datasetName": "SAG T2 CERV", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, + {"datasetName": "SAG T2 DORSAL", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, + + {"datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, + {"datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, + {"datasetName": "t2_tse_rr_p2_sag_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, + {"datasetName": "t2_tse_sag_384_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, + {"datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260_comp_ad", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, + {"datasetName": "t2_tse_sag_p2_comp_ad", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, + {"datasetName": "SAG T2 TSE MOELLE", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, + + + + {"datasetName": "t1_fl3d_sag_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_sag_p2_iso mt", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_tra_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_tra_p2_iso mt", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_we_tra_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_we_tra_p2_iso mt tr38", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_we_tra_p2_iso mt tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_we_tra_p2_iso mt_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_we_tra_p2_iso tr38", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat_mt", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38_sans_sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_we_tra_p2_iso_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_we_tra_p2_m0_iso tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, + {"datasetName": "t1_fl3d_we_tra_p2_mt_iso tr38", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, + + + {"datasetName": "3d_t1_mprage_sag", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, + {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, + {"datasetName": "t1_mprage_sag_p2_iso cerv", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, + {"datasetName": "t1_mprage_sag_p2_iso cerv_rr", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, + {"datasetName": "t1_mprage_sag_p2_iso ofsep", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, + {"datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_axial", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, + {"datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_cor", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, + {"datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_tra", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, + {"datasetName": "t1_mprage_sag_p2_iso ofsep_rr", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, + {"datasetName": "T1 MPRAGE CERV", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"} + + ], + "dcm2niix":"~/softs/miniconda3/bin/dcm2niix", + "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", "dcm2niix_options": { "bids_format": true, "anon_bids": true, @@ -306,3 +77,5 @@ "verbose": true } } + + From 51a662ad9cf48030b425868dfca41f0c58d88698 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 21 May 2024 14:22:54 +0200 Subject: [PATCH 77/86] [LINT]: black linting these files --- s2b_example_config_EMISEP_long.json | 367 ++++++++++++++++++++++------ shanoir2bids_heudiconv.py | 116 +++++---- 2 files changed, 372 insertions(+), 111 deletions(-) diff --git a/s2b_example_config_EMISEP_long.json b/s2b_example_config_EMISEP_long.json index c7bfb4a..d000628 100644 --- a/s2b_example_config_EMISEP_long.json +++ b/s2b_example_config_EMISEP_long.json @@ -1,70 +1,299 @@ { - "study_name": "EMISEP", - "subjects": ["08-74"], - "session": "08-74 MO ENCEPHALE", - "data_to_bids": - [ - {"datasetName": "3D T1 MPRAGE", "bidsDir": "brain", "bidsName": "T1w", "bidsSession": "M00"}, - {"datasetName": "*3d flair*", "bidsDir": "brain", "bidsName": "FLAIR", "bidsSession": "M00"}, - {"datasetName": "*c1c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", "bidsSession": "M00"}, - {"datasetName": "*c1-c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", "bidsSession": "M00"}, - - {"datasetName": "*c4c7*", "bidsDir": "spinalCord", "bidsName": "t2ax_C4C7", "bidsSession": "M00"}, - {"datasetName": "*c4-c7*", "bidsDir": "spinalCord", "bidsName": "t2ax_C4C7", "bidsSession": "M00"}, - - - {"datasetName": "t2_tse_rr_p2_sag", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "t2_tse_rr_p2_sag2.5 te 81", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_384", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_p2", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "SAG T2 CERV", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - {"datasetName": "SAG T2 DORSAL", "bidsDir": "spinalCord", "bidsName": "t2Sag", "bidsSession": "M00"}, - - {"datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "t2_tse_rr_p2_sag_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_384_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260_comp_ad", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "t2_tse_sag_p2_comp_ad", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - {"datasetName": "SAG T2 TSE MOELLE", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", "bidsSession": "M00"}, - - - - {"datasetName": "t1_fl3d_sag_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_sag_p2_iso mt", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_tra_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_tra_p2_iso mt", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso mt tr38", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso mt tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso mt_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso tr38", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat_mt", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38_sans_sat", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_iso_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_m0_iso tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", "bidsSession": "M00"}, - {"datasetName": "t1_fl3d_we_tra_p2_mt_iso tr38", "bidsDir": "spinalCord", "bidsName": "mt1", "bidsSession": "M00"}, - - - {"datasetName": "3d_t1_mprage_sag", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso cerv", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso cerv_rr", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso ofsep", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_axial", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_cor", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_tra", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "t1_mprage_sag_p2_iso ofsep_rr", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"}, - {"datasetName": "T1 MPRAGE CERV", "bidsDir": "spinalCord", "bidsName": "t1Sag", "bidsSession": "M00"} - - ], - "dcm2niix":"~/softs/miniconda3/bin/dcm2niix", - "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", + "study_name": "EMISEP", + "subjects": ["08-74"], + "session": "08-74 MO ENCEPHALE", + "data_to_bids": [ + { + "datasetName": "3D T1 MPRAGE", + "bidsDir": "brain", + "bidsName": "T1w", + "bidsSession": "M00", + }, + { + "datasetName": "*3d flair*", + "bidsDir": "brain", + "bidsName": "FLAIR", + "bidsSession": "M00", + }, + { + "datasetName": "*c1c3*", + "bidsDir": "spinalCord", + "bidsName": "t2ax_C1C3", + "bidsSession": "M00", + }, + { + "datasetName": "*c1-c3*", + "bidsDir": "spinalCord", + "bidsName": "t2ax_C1C3", + "bidsSession": "M00", + }, + { + "datasetName": "*c4c7*", + "bidsDir": "spinalCord", + "bidsName": "t2ax_C4C7", + "bidsSession": "M00", + }, + { + "datasetName": "*c4-c7*", + "bidsDir": "spinalCord", + "bidsName": "t2ax_C4C7", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_rr_p2_sag", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_rr_p2_sag2.5 te 81", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_384", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_p2", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "SAG T2 CERV", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "SAG T2 DORSAL", + "bidsDir": "spinalCord", + "bidsName": "t2Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp_sp", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_rr_p2_sag_comp_sp", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_384_comp_sp", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260_comp_ad", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t2_tse_sag_p2_comp_ad", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "SAG T2 TSE MOELLE", + "bidsDir": "spinalCord", + "bidsName": "t2Sag-COMP", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_sag_p2_iso", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_sag_p2_iso mt", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_tra_p2_iso", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_tra_p2_iso mt", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso mt tr38", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso mt tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso mt_tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso tr38", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat_mt", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38_sans_sat", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_iso_tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_m0_iso tr38 sans sat", + "bidsDir": "spinalCord", + "bidsName": "mt0", + "bidsSession": "M00", + }, + { + "datasetName": "t1_fl3d_we_tra_p2_mt_iso tr38", + "bidsDir": "spinalCord", + "bidsName": "mt1", + "bidsSession": "M00", + }, + { + "datasetName": "3d_t1_mprage_sag", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso cerv", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso cerv_rr", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso ofsep", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_axial", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_cor", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_tra", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "t1_mprage_sag_p2_iso ofsep_rr", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + { + "datasetName": "T1 MPRAGE CERV", + "bidsDir": "spinalCord", + "bidsName": "t1Sag", + "bidsSession": "M00", + }, + ], + "dcm2niix": "~/softs/miniconda3/bin/dcm2niix", + "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", "dcm2niix_options": { "bids_format": true, "anon_bids": true, @@ -74,8 +303,6 @@ "has_private": false, "ignore_deriv": false, "single_file": false, - "verbose": true - } + "verbose": true, + }, } - - diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index c31fadc..19c7427 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -4,8 +4,7 @@ The script is made to run for every project given some information provided by the user into a ".json" configuration file. More details regarding the configuration file in the Readme.md""" # Script to download and BIDS-like organize data on Shanoir using "shanoir_downloader.py" developed by Arthur Masson -# @Author: Malo Gaubert , Quentin Duché -# @Date: 24 Juin 2022 + import os from os.path import join as opj, splitext as ops, exists as ope, dirname as opd @@ -173,7 +172,9 @@ def read_json_config_file(json_file): def generate_heuristic_file( - shanoir2bids_dict: object, path_heuristic_file: object, output_type='("dicom","nii.gz")' + shanoir2bids_dict: object, + path_heuristic_file: object, + output_type='("dicom","nii.gz")', ) -> None: """Generate heudiconv heuristic.py file from shanoir2bids mapping dict Parameters @@ -181,9 +182,9 @@ def generate_heuristic_file( shanoir2bids_dict : path_heuristic_file : path of the python heuristic file (.py) """ - if output_type == 'dicom': + if output_type == "dicom": outtype = '("dicom",)' - elif output_type == 'nifti': + elif output_type == "nifti": outtype = '("nii.gz",)' else: outtype = '("dicom","nii.gz")' @@ -253,7 +254,9 @@ def __init__(self): self.shanoir_study_id = None # Shanoir study ID self.shanoir_session_id = None # Shanoir study ID self.shanoir_file_type = SHANOIR_FILE_TYPE_DICOM # Download File Type (DICOM) - self.output_file_type = DEFAULT_SHANOIR_FILE_TYPE # Default Export File Type (NIFTI) + self.output_file_type = ( + DEFAULT_SHANOIR_FILE_TYPE # Default Export File Type (NIFTI) + ) self.json_config_file = None self.list_fars = [] # List of substrings to edit in subjects names self.dl_dir = None # download directory, where data will be stored @@ -261,6 +264,7 @@ def __init__(self): self.n_seq = 0 # Number of sequences in the shanoir2bids_dict self.log_fn = None self.dcm2niix_path = None # Path to the dcm2niix the user wants to use + self.actual_dcm2niix_path = shutil.which("dcm2niix") self.dcm2niix_opts = None # Options to add to the dcm2niix call self.date_from = None self.date_to = None @@ -299,7 +303,7 @@ def set_json_config_file(self, json_file): self.set_date_to(date_to=date_to) def set_output_file_type(self, outfile_type): - if outfile_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, 'both']: + if outfile_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, "both"]: self.output_file_type = outfile_type else: sys.exit("Unknown output file type {}".format(outfile_type)) @@ -372,14 +376,19 @@ def set_log_filename(self): ) self.log_fn = opj(self.dl_dir, basename) - def toggle_longitudinal_version(self): - self.longitudinal = True - def switch_to_automri_format(self): self.to_automri_format = True - def add_series_number_suffix(self): - self.add_sns = True + def toggle_longitudinal_version(self): + self.longitudinal = True + + def is_correct_dcm2niix(self): + current_version = Path(self.actual_dcm2niix_path) + config_version = Path(self.dcm2niix_path) + if current_version is not None and config_version is not None: + return config_version.samefile(current_version) + else: + return False def configure_parser(self): """ @@ -388,8 +397,13 @@ def configure_parser(self): self.parser = shanoir_downloader.create_arg_parser() shanoir_downloader.add_username_argument(self.parser) shanoir_downloader.add_domain_argument(self.parser) - self.parser.add_argument('-f', '--format', default='dicom', choices=['dicom'], - help='The format to download.') + self.parser.add_argument( + "-f", + "--format", + default="dicom", + choices=["dicom"], + help="The format to download.", + ) shanoir_downloader.add_output_folder_argument(self.parser) shanoir_downloader.add_configuration_arguments(self.parser) shanoir_downloader.add_search_arguments(self.parser) @@ -413,7 +427,6 @@ def download_subject(self, subject_to_search): # temporary directory containing dowloaded DICOM.zip files with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_dicom: with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_archive: - print(tmp_archive) # Loop on each sequence defined in the dictionary for seq in range(self.n_seq): # Isolate elements that are called many times @@ -580,7 +593,9 @@ def download_subject(self, subject_to_search): mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" ) as heuristic_file: # Generate Heudiconv heuristic file from configuration.json mapping - generate_heuristic_file(bids_mapping, heuristic_file.name, output_type=self.output_file_type) + generate_heuristic_file( + bids_mapping, heuristic_file.name, output_type=self.output_file_type + ) with tempfile.NamedTemporaryFile( mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" ) as dcm2niix_config_file: @@ -601,8 +616,39 @@ def download_subject(self, subject_to_search): if self.longitudinal: workflow_params["session"] = bids_seq_session + if self.to_automri_format: + workflow_params["bids_options"] = None workflow(**workflow_params) + if self.to_automri_format: + # horrible hack to adapt to automri ontology + dicoms = glob( + opj(self.dl_dir, str(self.shanoir_study_id), "**", "*.dcm"), + recursive=True, + ) + niftis = glob( + opj( + self.dl_dir, + str(self.shanoir_study_id), + "**", + "*.nii.gz", + ), + recursive=True, + ) + export_files = dicoms + niftis + to_modify_files = [f for f in export_files if not ".git" in f] + for f in to_modify_files: + new_file = f.replace("/" + subject_id + "/", "/") + new_file = new_file.replace("sub-", "su_") + os.system("git mv " + f + " " + new_file) + from datalad.api import save + + save( + path=opj(self.dl_dir, str(self.shanoir_study_id)), + recursive=True, + message="reformat into automri standart", + ) + fp.close() def download(self): @@ -637,13 +683,7 @@ def main(): default="shanoir.irisa.fr", help="The shanoir domain to query.", ) - # parser.add_argument( - # "-f", - # "--format", - # default="dicom", - # choices=["dicom"], - # help="The format to download.", - # ) + parser.add_argument( "--outformat", default="both", @@ -667,16 +707,10 @@ def main(): help="Toggle longitudinal approach.", ) - # parser.add_argument( - # "-a", "--automri", action="store_true", help="Switch to automri file tree." - # ) - # parser.add_argument( - # "-A", - # "--add_sns", - # action="store_true", - # help="Add series number suffix (compatible with -a)", - # ) - # Parse arguments + parser.add_argument( + "-a", "--automri", action="store_true", help="Switch to automri file tree." + ) + args = parser.parse_args() # Start configuring the DownloadShanoirDatasetToBids class instance @@ -693,14 +727,14 @@ def main(): if args.longitudinal: stb.toggle_longitudinal_version() - # if args.automri: - # stb.switch_to_automri_format() - # if args.add_sns: - # if not args.automri: - # print("Warning : -A option is only compatible with -a option.") - # stb.add_series_number_suffix() - - stb.download() + if args.automri: + stb.switch_to_automri_format() + if not stb.is_correct_dcm2niix(): + print( + f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}" + ) + else: + stb.download() if __name__ == "__main__": From 3a84b3d7f3738e3ba7328e5e9eba7bcdeefd3a2e Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Wed, 22 May 2024 14:58:17 +0200 Subject: [PATCH 78/86] [FIX] fixed typo introduced by black linting in json file --- s2b_example_config_EMISEP_long.json | 102 ++++++++++++++-------------- 1 file changed, 51 insertions(+), 51 deletions(-) diff --git a/s2b_example_config_EMISEP_long.json b/s2b_example_config_EMISEP_long.json index d000628..e22a037 100644 --- a/s2b_example_config_EMISEP_long.json +++ b/s2b_example_config_EMISEP_long.json @@ -7,290 +7,290 @@ "datasetName": "3D T1 MPRAGE", "bidsDir": "brain", "bidsName": "T1w", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "*3d flair*", "bidsDir": "brain", "bidsName": "FLAIR", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "*c1c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "*c1-c3*", "bidsDir": "spinalCord", "bidsName": "t2ax_C1C3", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "*c4c7*", "bidsDir": "spinalCord", "bidsName": "t2ax_C4C7", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "*c4-c7*", "bidsDir": "spinalCord", "bidsName": "t2ax_C4C7", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_rr_p2_sag", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_rr_p2_sag2.5 te 81", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_384", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_p2", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "SAG T2 CERV", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "SAG T2 DORSAL", "bidsDir": "spinalCord", "bidsName": "t2Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_rr_p2_sag2.5 te 81_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_rr_p2_sag_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_384_comp_sp", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_p2_2mm_te 81 fov 260_comp_ad", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t2_tse_sag_p2_comp_ad", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "SAG T2 TSE MOELLE", "bidsDir": "spinalCord", "bidsName": "t2Sag-COMP", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_sag_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_sag_p2_iso mt", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_tra_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_tra_p2_iso mt", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso mt tr38", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso mt tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso mt_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso tr38", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso tr38 mt sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso tr38 sans sat_mt", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso_mt_tr38_sans_sat", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_iso_tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_m0_iso tr38 sans sat", "bidsDir": "spinalCord", "bidsName": "mt0", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_fl3d_we_tra_p2_mt_iso tr38", "bidsDir": "spinalCord", "bidsName": "mt1", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "3d_t1_mprage_sag", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso cerv", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso cerv_rr", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso ofsep", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_axial", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_cor", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso ofsep_mpr_tra", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "t1_mprage_sag_p2_iso ofsep_rr", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", + "bidsSession": "M00" }, { "datasetName": "T1 MPRAGE CERV", "bidsDir": "spinalCord", "bidsName": "t1Sag", - "bidsSession": "M00", - }, + "bidsSession": "M00" + } ], "dcm2niix": "~/softs/miniconda3/bin/dcm2niix", "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", @@ -303,6 +303,6 @@ "has_private": false, "ignore_deriv": false, "single_file": false, - "verbose": true, - }, + "verbose": true + } } From 898ee62886a1868624c52dbc49edd1de414dc1a5 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Fri, 22 Mar 2024 16:58:05 +0100 Subject: [PATCH 79/86] [ENH]: DICOM to NIFTI BIDS conversion with heuristic generated from configuration.json file. Configuration file was slightly modified to handle dcm2niix options following nipype DC2NIIX attributes --- shanoir2bidsv1.py | 672 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 672 insertions(+) create mode 100755 shanoir2bidsv1.py diff --git a/shanoir2bidsv1.py b/shanoir2bidsv1.py new file mode 100755 index 0000000..7016425 --- /dev/null +++ b/shanoir2bidsv1.py @@ -0,0 +1,672 @@ +#!/usr/bin/env python3 +DESCRIPTION = """ +shanoir2bids.py is a script that allows to download a Shanoir dataset and organise it as a BIDS data structure. + The script is made to run for every project given some information provided by the user into a ".json" + configuration file. More details regarding the configuration file in the Readme.md""" +# Script to download and BIDS-like organize data on Shanoir using "shanoir_downloader.py" developed by Arthur Masson +# @Author: Malo Gaubert , Quentin Duché +# @Date: 24 Juin 2022 +import os +import sys +import zipfile +import json +import shutil +import shanoir_downloader +from dotenv import load_dotenv +from pathlib import Path +from os.path import join as opj, splitext as ops, exists as ope, dirname as opd +from glob import glob +from time import time +import datetime +from dateutil import parser + +from heudiconv.main import workflow + + +# Load environment variables +load_dotenv(dotenv_path=opj(opd(__file__), ".env")) + + +def banner_msg(msg): + """ + Print a message framed by a banner of "*" characters + :param msg: + """ + banner = "*" * (len(msg) + 6) + print(banner + "\n* ", msg, " *\n" + banner) + + +# Keys for json configuration file +K_JSON_STUDY_NAME = "study_name" +K_JSON_L_SUBJECTS = "subjects" +K_JSON_SESSION = "session" +K_JSON_DATA_DICT = "data_to_bids" +K_JSON_FIND_AND_REPLACE = "find_and_replace_subject" +K_DCM2NIIX_PATH = "dcm2niix" +K_DCM2NIIX_OPTS = "dcm2niix_options" +K_FIND = "find" +K_REPLACE = "replace" +K_JSON_DATE_FROM = ( + "date_from" # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] +) +K_JSON_DATE_TO = ( + "date_to" # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] +) +LIST_MANDATORY_KEYS_JSON = [K_JSON_STUDY_NAME, K_JSON_L_SUBJECTS, K_JSON_DATA_DICT] +LIST_AUTHORIZED_KEYS_JSON = LIST_MANDATORY_KEYS_JSON + [ + K_DCM2NIIX_PATH, + K_DCM2NIIX_OPTS, + K_JSON_DATE_FROM, + K_JSON_DATE_TO, + K_JSON_SESSION, +] + +# Define keys for data dictionary +K_BIDS_NAME = "bidsName" +K_BIDS_DIR = "bidsDir" +K_BIDS_SES = "bidsSession" +K_DS_NAME = "datasetName" + +# Define Extensions that are dealt so far by (#todo : think of other possible extensions ?) +NIFTI = ".nii" +NIIGZ = ".nii.gz" +JSON = ".json" +BVAL = ".bval" +BVEC = ".bvec" +DCM = ".dcm" + +# Shanoir parameters +SHANOIR_FILE_TYPE_NIFTI = "nifti" +SHANOIR_FILE_TYPE_DICOM = "dicom" +DEFAULT_SHANOIR_FILE_TYPE = SHANOIR_FILE_TYPE_NIFTI + +# Define error and warning messages when call to dcm2niix is not well configured in the json file +DCM2NIIX_ERR_MSG = """ERROR !! +Conversion from DICOM to nifti can not be performed. +Please provide path to your favorite dcm2niix version in your Shanoir2BIDS .json configuration file. +Add key "{key}" with the absolute path to dcm2niix version to the following file : """ +DCM2NIIX_WARN_MSG = """WARNING. You did not provide any option to the dcm2niix call. +If you want to do so, add key "{key}" to you Shanoir2BIDS configuration file :""" + + +def check_date_format(date_to_format): + # TRUE FORMAT should be: date_format = 'Y-m-dTH:M:SZ' + try: + parser.parse(date_to_format) + # If the date validation goes wrong + except ValueError: + print( + "Incorrect data format, should be YYYY-MM-DDTHH:MM:SSZ (for example: 2020-02-19T00:00:00Z)" + ) + + +def read_json_config_file(json_file): + """ + Reads a json configuration file and checks whether mandatory keys for specifying the transformation from a + Shanoir dataset to a BIDS dataset is present. + :param json_file: str, path to a json configuration file + :return: + """ + f = open(json_file) + data = json.load(f) + # Check keys + for key in data.keys(): + if not key in LIST_AUTHORIZED_KEYS_JSON: + print('Unknown key "{}" for data dictionary'.format(key)) + for key in LIST_MANDATORY_KEYS_JSON: + if not key in data.keys(): + sys.exit('Error, missing key "{}" in data dictionary'.format(key)) + + # Sets the mandatory fields for the instance of the class + study_id = data[K_JSON_STUDY_NAME] + subjects = data[K_JSON_L_SUBJECTS] + data_dict = data[K_JSON_DATA_DICT] + + # Default non-mandatory options + list_fars = [] + dcm2niix_path = None + dcm2niix_opts = None + date_from = "*" + date_to = "*" + session_id = "*" + + if K_JSON_FIND_AND_REPLACE in data.keys(): + list_fars = data[K_JSON_FIND_AND_REPLACE] + if K_DCM2NIIX_PATH in data.keys(): + dcm2niix_path = data[K_DCM2NIIX_PATH] + if K_DCM2NIIX_OPTS in data.keys(): + dcm2niix_opts = data[K_DCM2NIIX_OPTS] + if K_JSON_DATE_FROM in data.keys(): + if data[K_JSON_DATE_FROM] == "": + data_from = "*" + else: + date_from = data[K_JSON_DATE_FROM] + check_date_format(date_from) + if K_JSON_DATE_TO in data.keys(): + if data[K_JSON_DATE_TO] == "": + data_to = "*" + else: + date_to = data[K_JSON_DATE_TO] + check_date_format(date_to) + if K_JSON_SESSION in data.keys(): + session_id = data[K_JSON_SESSION] + + # Close json file and return + f.close() + return ( + study_id, + subjects, + session_id, + data_dict, + list_fars, + dcm2niix_path, + dcm2niix_opts, + date_from, + date_to, + ) + + +def generate_heuristic_file( + shanoir2bids_dict: object, path_heuristic_file: object +) -> None: + """Generate heudiconv heuristic.py file from shanoir2bids mapping dict + Parameters + ---------- + shanoir2bids_dict : + path_heuristic_file : path of the python heuristic file (.py) + """ + heuristic = f"""from heudiconv.heuristics.reproin import create_key + +def create_bids_key(dataset): + template = create_key(subdir=dataset['bidsDir'],file_suffix=dataset['bidsName'],outtype=("dicom","nii.gz")) + return template + +def get_dataset_to_key_mapping(shanoir2bids): + dataset_to_key = dict() + for dataset in shanoir2bids: + template = create_bids_key(dataset) + dataset_to_key[dataset['datasetName']] = template + return dataset_to_key + + +def infotodict(seqinfo): + + info = dict() + shanoir2bids = {shanoir2bids_dict} + + dataset_to_key = get_dataset_to_key_mapping(shanoir2bids) + for seq in seqinfo: + if seq.series_description in dataset_to_key.keys(): + key = dataset_to_key[seq.series_description] + if key in info.keys(): + info[key].append(seq.series_id) + else: + info[key] = [seq.series_id] + return info +""" + + with open(path_heuristic_file, "w", encoding="utf-8") as file: + file.write(heuristic) + file.close() + pass + + +class DownloadShanoirDatasetToBIDS: + """ + class that handles the downloading of shanoir data set and the reformatting as a BIDS data structure + """ + + def __init__(self): + """ + Initialize the class instance + """ + self.shanoir_subjects = None # List of Shanoir subjects + self.shanoir2bids_dict = ( + None # Dictionary specifying how to reformat data into BIDS structure + ) + self.shanoir_username = None # Shanoir username + self.shanoir_study_id = None # Shanoir study ID + self.shanoir_session_id = None # Shanoir study ID + self.shanoir_file_type = ( + DEFAULT_SHANOIR_FILE_TYPE # Default download type (nifti/dicom) + ) + self.json_config_file = None + self.list_fars = [] # List of substrings to edit in subjects names + self.dl_dir = None # download directory, where data will be stored + self.parser = None # Shanoir Downloader Parser + self.n_seq = 0 # Number of sequences in the shanoir2bids_dict + self.log_fn = None + self.dcm2niix_path = None # Path to the dcm2niix the user wants to use + self.dcm2niix_opts = None # Options to add to the dcm2niix call + self.dcm2niix_config_file = None # .json file to store dcm2niix options + self.date_from = None + self.date_to = None + self.longitudinal = False + self.to_automri_format = ( + False # Special filenames for automri (close to BIDS format) + ) + self.add_sns = False # Add series number suffix to filename + + def set_json_config_file(self, json_file): + """ + Sets the configuration for the download through a json file + :param json_file: str, path to the json_file + """ + self.json_config_file = json_file + ( + study_id, + subjects, + session_id, + data_dict, + list_fars, + dcm2niix_path, + dcm2niix_opts, + date_from, + date_to, + ) = read_json_config_file(json_file=json_file) + self.set_shanoir_study_id(study_id=study_id) + self.set_shanoir_subjects(subjects=subjects) + self.set_shanoir_session_id(session_id=session_id) + self.set_shanoir2bids_dict(data_dict=data_dict) + self.set_shanoir_list_find_and_replace(list_fars=list_fars) + self.set_dcm2niix_parameters( + dcm2niix_path=dcm2niix_path, dcm2niix_opts=dcm2niix_opts + ) + self.set_date_from(date_from=date_from) + self.set_date_to(date_to=date_to) + + def set_shanoir_file_type(self, shanoir_file_type): + if shanoir_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI]: + self.shanoir_file_type = shanoir_file_type + else: + sys.exit("Unknown shanoir file type {}".format(shanoir_file_type)) + + def set_shanoir_study_id(self, study_id): + self.shanoir_study_id = study_id + + def set_shanoir_username(self, shanoir_username): + self.shanoir_username = shanoir_username + + def set_shanoir_domaine(self, shanoir_domaine): + self.shanoir_domaine = shanoir_domaine + + def set_shanoir_subjects(self, subjects): + self.shanoir_subjects = subjects + + def set_shanoir_session_id(self, session_id): + self.shanoir_session_id = session_id + + def set_shanoir_list_find_and_replace(self, list_fars): + self.list_fars = list_fars + + def set_dcm2niix_parameters(self, dcm2niix_path, dcm2niix_opts): + self.dcm2niix_path = dcm2niix_path + self.dcm2niix_opts = dcm2niix_opts + + def set_dcm2niix_config_files(self, path_dcm2niix_options_files): + self.dcm2niix_config_file = path_dcm2niix_options_files + # Serializing json + json_object = json.dumps(self.dcm2niix_opts, indent=4) + with open(self.dcm2niix_config_file, "w") as file: + file.write(json_object) + + def set_date_from(self, date_from): + self.date_from = date_from + + def set_date_to(self, date_to): + self.date_to = date_to + + def set_shanoir2bids_dict(self, data_dict): + self.shanoir2bids_dict = data_dict + self.n_seq = len(self.shanoir2bids_dict) + + def set_download_directory(self, dl_dir): + if dl_dir is None: + # Create a default download directory + dt = datetime.datetime.now().strftime("%Y_%m_%d_at_%Hh%Mm%Ss") + self.dl_dir = "_".join( + ["shanoir2bids", "download", self.shanoir_study_id, dt] + ) + print( + "A NEW DEFAULT directory is created as you did not provide a download directory (-of option)\n\t" + + self.dl_dir + ) + else: + self.dl_dir = dl_dir + # Create directory if it does not exist + if not ope(self.dl_dir): + Path(self.dl_dir).mkdir(parents=True, exist_ok=True) + self.set_log_filename() + + def set_heuristic_file(self, path_heuristic_file): + if path_heuristic_file is None: + print("TO BE DONE") + else: + self.heuristic_file = path_heuristic_file + + def set_log_filename(self): + curr_time = datetime.datetime.now() + basename = "shanoir_downloader_{:04}{:02}{:02}_{:02}{:02}{:02}.log".format( + curr_time.year, + curr_time.month, + curr_time.day, + curr_time.hour, + curr_time.minute, + curr_time.second, + ) + self.log_fn = opj(self.dl_dir, basename) + + def toggle_longitudinal_version(self): + self.longitudinal = True + + def switch_to_automri_format(self): + self.to_automri_format = True + + def add_series_number_suffix(self): + self.add_sns = True + + def configure_parser(self): + """ + Configure the parser and the configuration of the shanoir_downloader + """ + self.parser = shanoir_downloader.create_arg_parser() + shanoir_downloader.add_common_arguments(self.parser) + shanoir_downloader.add_configuration_arguments(self.parser) + shanoir_downloader.add_search_arguments(self.parser) + shanoir_downloader.add_ids_arguments(self.parser) + + def download_subject(self, subject_to_search): + """ + For a single subject + 1. Downloads the Shanoir datasets + 2. Reorganises the Shanoir dataset as BIDS format as defined in the json configuration file provided by user + :param subject_to_search: + :return: + """ + banner_msg("Downloading subject " + subject_to_search) + + # Open log file to write the steps of processing (downloading, renaming...) + fp = open(self.log_fn, "a") + + # Real Shanoir2Bids mapping ( deal with search terms are included in datasetName field) + bids_mapping = [] + + # Loop on each sequence defined in the dictionary + for seq in range(self.n_seq): + # Isolate elements that are called many times + shanoir_seq_name = self.shanoir2bids_dict[seq][ + K_DS_NAME + ] # Shanoir sequence name (OLD) + bids_seq_subdir = self.shanoir2bids_dict[seq][ + K_BIDS_DIR + ] # Sequence BIDS subdirectory name (NEW) + bids_seq_name = self.shanoir2bids_dict[seq][ + K_BIDS_NAME + ] # Sequence BIDS nickname (NEW) + if self.longitudinal: + bids_seq_session = self.shanoir2bids_dict[seq][ + K_BIDS_SES + ] # Sequence BIDS nickname (NEW) + + # Print message concerning the sequence that is being downloaded + print( + "\t-", + bids_seq_name, + subject_to_search, + "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", + ) + + # Initialize the parser + search_txt = ( + "studyName:" + + self.shanoir_study_id.replace(" ", "?") + + " AND datasetName:" + + shanoir_seq_name.replace(" ", "?") + + " AND subjectName:" + + subject_to_search.replace(" ", "?") + + " AND examinationComment:" + + self.shanoir_session_id.replace(" ", "*") + + " AND examinationDate:[" + + self.date_from + + " TO " + + self.date_to + + "]" + ) + + args = self.parser.parse_args( + [ + "-u", + self.shanoir_username, + "-d", + self.shanoir_domaine, + "-of", + self.dl_dir, + "-em", + "-st", + search_txt, + "-s", + "200", + "-f", + self.shanoir_file_type, + "-so", + "id,ASC", + "-t", + "500", + ] + ) # Increase time out for heavy files + + config = shanoir_downloader.initialize(args) + response = shanoir_downloader.solr_search(config, args) + + # From response, process the data + # Print the number of items found and a list of these items + if response.status_code == 200: + # Invoke shanoir_downloader to download all the data + shanoir_downloader.download_search_results(config, args, response) + + if len(response.json()["content"]) == 0: + warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns +a result on the website. +Search Text : "{}" \n""".format( + search_txt + ) + print(warn_msg) + fp.write(warn_msg) + else: + for item in response.json()["content"]: + # Define subject_id + su_id = item["subjectName"] + # If the user has defined a list of edits to subject names... then do the find and replace + for far in self.list_fars: + su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) + + # ID of the subject (sub-*) + subject_id = su_id + # correct BIDS mapping of the searched dataset + bids_seq_mapping = { + "datasetName": item["datasetName"], + "bidsDir": bids_seq_subdir, + "bidsName": bids_seq_name, + "bids_subject_id": subject_id, + } + + if self.longitudinal: + bids_seq_mapping["bids_session_id"] = bids_seq_session + else: + bids_seq_mapping["bids_session_id"] = None + + bids_mapping.append(bids_seq_mapping) + + # Write the information on the data in the log file + fp.write("- datasetId = " + str(item["datasetId"]) + "\n") + fp.write(" -- studyName: " + item["studyName"] + "\n") + fp.write(" -- subjectName: " + item["subjectName"] + "\n") + fp.write(" -- session: " + item["examinationComment"] + "\n") + fp.write(" -- datasetName: " + item["datasetName"] + "\n") + fp.write( + " -- examinationDate: " + item["examinationDate"] + "\n" + ) + fp.write(" >> Downloading archive OK\n") + + # Create temp directory to make sure the directory is empty before + # TODO: Replace with temp directory ? + tmp_dir = opj(self.dl_dir, "temp_archive") + Path(tmp_dir).mkdir(parents=True, exist_ok=True) + + # Extract the downloaded archive + dl_archive = glob(opj(self.dl_dir, "*" + item["id"] + "*.zip"))[ + 0 + ] + with zipfile.ZipFile(dl_archive, "r") as zip_ref: + extraction_dir = opj(tmp_dir, item["id"]) + zip_ref.extractall(extraction_dir) + + fp.write( + " >> Extraction of all files from archive '" + + dl_archive + + " into " + + tmp_dir + + item["id"] + + "\n" + ) + + elif response.status_code == 204: + banner_msg("ERROR : No file found!") + fp.write(" >> ERROR : No file found!\n") + else: + banner_msg( + "ERROR : Returned by the request: status of the response = " + + response.status_code + ) + fp.write( + " >> ERROR : Returned by the request: status of the response = " + + str(response.status_code) + + "\n" + ) + # Generate Heudiconv heuristic file from .json mapping + generate_heuristic_file(bids_mapping, self.heuristic_file) + # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options + + if self.longitudinal: + workflow( + files=glob(opj(self.dl_dir, 'temp_archive', "*", "*.dcm"), recursive=True), + outdir=opj(self.dl_dir, "test"), + subjs=[subject_id], + session = bids_seq_session, + converter="dcm2niix", + heuristic=self.heuristic_file, + bids_options='--bids', + dcmconfig=self.dcm2niix_config_file, + datalad=True, + minmeta=True, + ) + else: + + workflow( + files=glob(opj(self.dl_dir,'temp_archive',"*", "*.dcm"), recursive=True), + outdir=opj(self.dl_dir, "test"), + subjs=[subject_id], + converter="dcm2niix", + heuristic=self.heuristic_file, + bids_options='--bids', + dcmconfig=self.dcm2niix_config_file, + datalad=True, + minmeta=True, + ) + fp.close() + + def download(self): + """ + Loop over the Shanoir subjects and go download the required datasets + :return: + """ + self.set_log_filename() + self.configure_parser() # Configure the shanoir_downloader parser + fp = open(self.log_fn, "w") + for subject_to_search in self.shanoir_subjects: + t_start_subject = time() + self.download_subject(subject_to_search=subject_to_search) + dur_min = int((time() - t_start_subject) // 60) + dur_sec = int((time() - t_start_subject) % 60) + end_msg = ( + "Downloaded dataset for subject " + + subject_to_search + + " in {}m{}s".format(dur_min, dur_sec) + ) + banner_msg(end_msg) + + +def main(): + # Parse argument for the script + parser = shanoir_downloader.create_arg_parser(description=DESCRIPTION) + # Use username and output folder arguments from shanoir_downloader + shanoir_downloader.add_username_argument(parser) + parser.add_argument( + "-d", + "--domain", + default="shanoir.irisa.fr", + help="The shanoir domain to query.", + ) + parser.add_argument( + "-f", + "--format", + default="dicom", + choices=["dicom"], + help="The format to download.", + ) + shanoir_downloader.add_output_folder_argument(parser=parser, required=False) + # Add the argument for the configuration file + parser.add_argument( + "-j", + "--config_file", + required=True, + help="Path to the .json configuration file specifying parameters for shanoir downloading.", + ) + parser.add_argument( + "-L", + "--longitudinal", + required=False, + action="store_true", + help="Toggle longitudinal approach.", + ) + # parser.add_argument( + # "-a", "--automri", action="store_true", help="Switch to automri file tree." + # ) + # parser.add_argument( + # "-A", + # "--add_sns", + # action="store_true", + # help="Add series number suffix (compatible with -a)", + # ) + # Parse arguments + args = parser.parse_args() + + # Start configuring the DownloadShanoirDatasetToBids class instance + stb = DownloadShanoirDatasetToBIDS() + stb.set_shanoir_username(args.username) + stb.set_shanoir_domaine(args.domain) + stb.set_json_config_file( + json_file=args.config_file + ) # path to json configuration file + stb.set_shanoir_file_type(shanoir_file_type=args.format) # Format (dicom or nifti) + stb.set_download_directory( + dl_dir=args.output_folder + ) # output folder (if None a default directory is created) + stb.set_heuristic_file(path_heuristic_file="/home/alpron/heuristic.py") + stb.set_dcm2niix_config_files( + path_dcm2niix_options_files="/home/alpron/dcm2niix_options.json" + ) + if args.longitudinal: + stb.toggle_longitudinal_version() + # if args.automri: + # stb.switch_to_automri_format() + # if args.add_sns: + # if not args.automri: + # print("Warning : -A option is only compatible with -a option.") + # stb.add_series_number_suffix() + + stb.download() + + +if __name__ == "__main__": + main() From ff5aee97383990e3299ac3bb45e0105388779f9e Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 26 Mar 2024 10:39:15 +0100 Subject: [PATCH 80/86] change main script name --- shanoir2bids_heudiconv.py | 447 +++++++++++++------------ shanoir2bidsv1.py | 672 -------------------------------------- 2 files changed, 218 insertions(+), 901 deletions(-) delete mode 100755 shanoir2bidsv1.py diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index 19c7427..e1d1487 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -4,7 +4,8 @@ The script is made to run for every project given some information provided by the user into a ".json" configuration file. More details regarding the configuration file in the Readme.md""" # Script to download and BIDS-like organize data on Shanoir using "shanoir_downloader.py" developed by Arthur Masson - +# @Author: Malo Gaubert , Quentin Duché +# @Date: 24 Juin 2022 import os from os.path import join as opj, splitext as ops, exists as ope, dirname as opd @@ -17,16 +18,12 @@ import tempfile from dateutil import parser import json -import logging import shutil import shanoir_downloader from dotenv import load_dotenv from heudiconv.main import workflow -# import loggger used in heudiconv workflow -from heudiconv.main import lgr - # Load environment variables load_dotenv(dotenv_path=opj(opd(__file__), ".env")) @@ -94,6 +91,14 @@ def banner_msg(msg): If you want to do so, add key "{key}" to you Shanoir2BIDS configuration file :""" +def create_tmp_directory(path_temporary_directory): + tmp_dir = Path(path_temporary_directory) + if tmp_dir.exists(): + shutil.rmtree(tmp_dir) + tmp_dir.mkdir(parents=True) + pass + + def check_date_format(date_to_format): # TRUE FORMAT should be: date_format = 'Y-m-dTH:M:SZ' try: @@ -171,9 +176,9 @@ def read_json_config_file(json_file): ) -def generate_heuristic_file( - shanoir2bids_dict: object, - path_heuristic_file: object, +def generate_bids_heuristic_file( + shanoir2bids_dict, + path_heuristic_file, output_type='("dicom","nii.gz")', ) -> None: """Generate heudiconv heuristic.py file from shanoir2bids mapping dict @@ -270,9 +275,10 @@ def __init__(self): self.date_to = None self.longitudinal = False self.to_automri_format = ( - False # Special filenames for automri (close to BIDS format) + False # Special filenames for automri (No longer used ! --> BIDS format) ) self.add_sns = False # Add series number suffix to filename + self.debug_mode = False # No debug mode by default def set_json_config_file(self, json_file): """ @@ -364,6 +370,18 @@ def set_download_directory(self, dl_dir): Path(self.dl_dir).mkdir(parents=True, exist_ok=True) self.set_log_filename() + def set_heuristic_file(self, path_heuristic_file): + if path_heuristic_file is None: + print(f"No heuristic file provided") + else: + filename, ext = ops(path_heuristic_file) + if ext != ".py": + print( + f"Provided heuristic file {path_heuristic_file} is not a .py file as expected" + ) + else: + self.heuristic_file = path_heuristic_file + def set_log_filename(self): curr_time = datetime.datetime.now() basename = "shanoir_downloader_{:04}{:02}{:02}_{:02}{:02}{:02}.log".format( @@ -376,9 +394,6 @@ def set_log_filename(self): ) self.log_fn = opj(self.dl_dir, basename) - def switch_to_automri_format(self): - self.to_automri_format = True - def toggle_longitudinal_version(self): self.longitudinal = True @@ -424,232 +439,202 @@ def download_subject(self, subject_to_search): # Real Shanoir2Bids mapping (handle case when solr search term are included) bids_mapping = [] - # temporary directory containing dowloaded DICOM.zip files - with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_dicom: - with tempfile.TemporaryDirectory(dir=self.dl_dir) as tmp_archive: - # Loop on each sequence defined in the dictionary - for seq in range(self.n_seq): - # Isolate elements that are called many times - shanoir_seq_name = self.shanoir2bids_dict[seq][ - K_DS_NAME - ] # Shanoir sequence name (OLD) - bids_seq_subdir = self.shanoir2bids_dict[seq][ - K_BIDS_DIR - ] # Sequence BIDS subdirectory name (NEW) - bids_seq_name = self.shanoir2bids_dict[seq][ - K_BIDS_NAME - ] # Sequence BIDS nickname (NEW) - if self.longitudinal: - bids_seq_session = self.shanoir2bids_dict[seq][ - K_BIDS_SES - ] # Sequence BIDS nickname (NEW) - - # Print message concerning the sequence that is being downloaded - print( - "\t-", - bids_seq_name, - subject_to_search, - "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", - ) - # Initialize the parser - search_txt = ( - "studyName:" - + self.shanoir_study_id.replace(" ", "?") - + " AND datasetName:" - + shanoir_seq_name.replace(" ", "?") - + " AND subjectName:" - + subject_to_search.replace(" ", "?") - + " AND examinationComment:" - + self.shanoir_session_id.replace(" ", "*") - + " AND examinationDate:[" - + self.date_from - + " TO " - + self.date_to - + "]" + # Manual temporary directories containing dowloaded DICOM.zip and extracted files + # (temporary directories that can be kept are not supported by pythn <3.1 + tmp_dicom = Path(self.dl_dir).joinpath("tmp_dicoms", subject_to_search) + tmp_archive = Path(self.dl_dir).joinpath( + "tmp_archived_dicoms", subject_to_search + ) + create_tmp_directory(tmp_archive) + create_tmp_directory(tmp_dicom) + + # Loop on each sequence defined in the dictionary + for seq in range(self.n_seq): + # Isolate elements that are called many times + shanoir_seq_name = self.shanoir2bids_dict[seq][ + K_DS_NAME + ] # Shanoir sequence name (OLD) + bids_seq_subdir = self.shanoir2bids_dict[seq][ + K_BIDS_DIR + ] # Sequence BIDS subdirectory name (NEW) + bids_seq_name = self.shanoir2bids_dict[seq][ + K_BIDS_NAME + ] # Sequence BIDS nickname (NEW) + if self.longitudinal: + bids_seq_session = self.shanoir2bids_dict[seq][ + K_BIDS_SES + ] # Sequence BIDS nickname (NEW) + + # Print message concerning the sequence that is being downloaded + print( + "\t-", + bids_seq_name, + subject_to_search, + "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", + ) + + # Initialize the parser + search_txt = ( + "studyName:" + + self.shanoir_study_id.replace(" ", "?") + + " AND datasetName:" + + shanoir_seq_name.replace(" ", "?") + + " AND subjectName:" + + subject_to_search.replace(" ", "?") + + " AND examinationComment:" + + self.shanoir_session_id.replace(" ", "*") + + " AND examinationDate:[" + + self.date_from + + " TO " + + self.date_to + + "]" + ) + + args = self.parser.parse_args( + [ + "-u", + self.shanoir_username, + "-d", + self.shanoir_domaine, + "-of", + str(tmp_archive), + "-em", + "-st", + search_txt, + "-s", + "200", + "-f", + self.shanoir_file_type, + "-so", + "id,ASC", + "-t", + "500", + ] + ) # Increase time out for heavy files + + config = shanoir_downloader.initialize(args) + response = shanoir_downloader.solr_search(config, args) + + # From response, process the data + # Print the number of items found and a list of these items + if response.status_code == 200: + # Invoke shanoir_downloader to download all the data + shanoir_downloader.download_search_results( + config, args, response + ) + + if len(response.json()["content"]) == 0: + warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns +a result on the website. +Search Text : "{}" \n""".format( + search_txt ) + print(warn_msg) + fp.write(warn_msg) + else: + for item in response.json()["content"]: + # Define subject_id + su_id = item["subjectName"] + # If the user has defined a list of edits to subject names... then do the find and replace + for far in self.list_fars: + su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) + # ID of the subject (sub-*) + subject_id = su_id + + # correct BIDS mapping of the searched dataset + bids_seq_mapping = { + "datasetName": item["datasetName"], + "bidsDir": bids_seq_subdir, + "bidsName": bids_seq_name, + "bids_subject_id": subject_id, + } + + if self.longitudinal: + bids_seq_mapping["bids_session_id"] = bids_seq_session + else: + bids_seq_session = None - args = self.parser.parse_args( - [ - "-u", - self.shanoir_username, - "-d", - self.shanoir_domaine, - "-of", - tmp_archive, - "-em", - "-st", - search_txt, - "-s", - "200", - "-f", - self.shanoir_file_type, - "-so", - "id,ASC", - "-t", - "500", - ] - ) # Increase time out for heavy files + bids_seq_mapping["bids_session_id"] = bids_seq_session - config = shanoir_downloader.initialize(args) - response = shanoir_downloader.solr_search(config, args) + bids_mapping.append(bids_seq_mapping) - # From response, process the data - # Print the number of items found and a list of these items - if response.status_code == 200: - # Invoke shanoir_downloader to download all the data - shanoir_downloader.download_search_results( - config, args, response + # Write the information on the data in the log file + fp.write("- datasetId = " + str(item["datasetId"]) + "\n") + fp.write(" -- studyName: " + item["studyName"] + "\n") + fp.write(" -- subjectName: " + item["subjectName"] + "\n") + fp.write(" -- session: " + item["examinationComment"] + "\n") + fp.write(" -- datasetName: " + item["datasetName"] + "\n") + fp.write( + " -- examinationDate: " + item["examinationDate"] + "\n" ) + fp.write(" >> Downloading archive OK\n") + + # Extract the downloaded archive + dl_archive = glob(opj(tmp_archive, "*" + item["id"] + "*.zip"))[ + 0 + ] + with zipfile.ZipFile(dl_archive, "r") as zip_ref: + extraction_dir = opj(tmp_dicom, item["id"]) + zip_ref.extractall(extraction_dir) - if len(response.json()["content"]) == 0: - warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns - a result on the website. - Search Text : "{}" \n""".format( - search_txt - ) - print(warn_msg) - fp.write(warn_msg) - else: - for item in response.json()["content"]: - # Define subject_id - su_id = item["subjectName"] - # If the user has defined a list of edits to subject names... then do the find and replace - for far in self.list_fars: - su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) - - # ID of the subject (sub-*) - subject_id = su_id - # correct BIDS mapping of the searched dataset - bids_seq_mapping = { - "datasetName": item["datasetName"], - "bidsDir": bids_seq_subdir, - "bidsName": bids_seq_name, - "bids_subject_id": subject_id, - } - - if self.longitudinal: - bids_seq_mapping[ - "bids_session_id" - ] = bids_seq_session - else: - bids_seq_mapping["bids_session_id"] = None - - bids_mapping.append(bids_seq_mapping) - - # Write the information on the data in the log file - fp.write( - "- datasetId = " + str(item["datasetId"]) + "\n" - ) - fp.write(" -- studyName: " + item["studyName"] + "\n") - fp.write( - " -- subjectName: " + item["subjectName"] + "\n" - ) - fp.write( - " -- session: " + item["examinationComment"] + "\n" - ) - fp.write( - " -- datasetName: " + item["datasetName"] + "\n" - ) - fp.write( - " -- examinationDate: " - + item["examinationDate"] - + "\n" - ) - fp.write(" >> Downloading archive OK\n") - - # Extract the downloaded archive - dl_archive = glob( - opj(tmp_archive, "*" + item["id"] + "*.zip") - )[0] - with zipfile.ZipFile(dl_archive, "r") as zip_ref: - extraction_dir = opj(tmp_dicom, item["id"]) - zip_ref.extractall(extraction_dir) - - fp.write( - " >> Extraction of all files from archive '" - + dl_archive - + " into " - + extraction_dir - + "\n" - ) - - elif response.status_code == 204: - banner_msg("ERROR : No file found!") - fp.write(" >> ERROR : No file found!\n") - else: - banner_msg( - "ERROR : Returned by the request: status of the response = " - + response.status_code - ) fp.write( - " >> ERROR : Returned by the request: status of the response = " - + str(response.status_code) + " >> Extraction of all files from archive '" + + dl_archive + + " into " + + extraction_dir + "\n" ) - # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options - with tempfile.NamedTemporaryFile( - mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" - ) as heuristic_file: - # Generate Heudiconv heuristic file from configuration.json mapping - generate_heuristic_file( - bids_mapping, heuristic_file.name, output_type=self.output_file_type + elif response.status_code == 204: + banner_msg("ERROR : No file found!") + fp.write(" >> ERROR : No file found!\n") + else: + banner_msg( + "ERROR : Returned by the request: status of the response = " + + response.status_code ) - with tempfile.NamedTemporaryFile( - mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" - ) as dcm2niix_config_file: - self.export_dcm2niix_config_options(dcm2niix_config_file.name) - workflow_params = { - "files": glob(opj(tmp_dicom, "*", "*.dcm"), recursive=True), - "outdir": opj(self.dl_dir, str(self.shanoir_study_id)), - "subjs": [subject_id], - "converter": "dcm2niix", - "heuristic": heuristic_file.name, - "bids_options": "--bids", - # "with_prov": True, - "dcmconfig": dcm2niix_config_file.name, - "datalad": True, - "minmeta": True, - "grouping": "all", # other options are too restrictive (tested on EMISEP) - } - - if self.longitudinal: - workflow_params["session"] = bids_seq_session - if self.to_automri_format: - workflow_params["bids_options"] = None - - workflow(**workflow_params) - if self.to_automri_format: - # horrible hack to adapt to automri ontology - dicoms = glob( - opj(self.dl_dir, str(self.shanoir_study_id), "**", "*.dcm"), - recursive=True, - ) - niftis = glob( - opj( - self.dl_dir, - str(self.shanoir_study_id), - "**", - "*.nii.gz", - ), - recursive=True, - ) - export_files = dicoms + niftis - to_modify_files = [f for f in export_files if not ".git" in f] - for f in to_modify_files: - new_file = f.replace("/" + subject_id + "/", "/") - new_file = new_file.replace("sub-", "su_") - os.system("git mv " + f + " " + new_file) - from datalad.api import save - - save( - path=opj(self.dl_dir, str(self.shanoir_study_id)), - recursive=True, - message="reformat into automri standart", - ) + fp.write( + " >> ERROR : Returned by the request: status of the response = " + + str(response.status_code) + + "\n" + ) + + # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options + with tempfile.NamedTemporaryFile( + mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" + ) as heuristic_file: + # Generate Heudiconv heuristic file from configuration.json mapping + generate_bids_heuristic_file( + bids_mapping, heuristic_file.name, output_type=self.output_file_type + ) + with tempfile.NamedTemporaryFile( + mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" + ) as dcm2niix_config_file: + self.export_dcm2niix_config_options(dcm2niix_config_file.name) + workflow_params = { + "files": glob(opj(tmp_dicom, "*", "*.dcm"), recursive=True), + "outdir": opj(self.dl_dir, str(self.shanoir_study_id)), + "subjs": [subject_id], + "converter": "dcm2niix", + "heuristic": heuristic_file.name, + "bids_options": "--bids", + # "with_prov": True, + "dcmconfig": dcm2niix_config_file.name, + "datalad": True, + "minmeta": True, + "grouping": "all", # other options are too restrictive (tested on EMISEP) + } + + if self.longitudinal: + workflow_params["session"] = bids_seq_session + + workflow(**workflow_params) + fp.close() + if not self.debug_mode: + shutil.rmtree(tmp_archive.parent) + shutil.rmtree(tmp_dicom.parent) - fp.close() def download(self): """ @@ -706,9 +691,11 @@ def main(): action="store_true", help="Toggle longitudinal approach.", ) - parser.add_argument( - "-a", "--automri", action="store_true", help="Switch to automri file tree." + "--debug", + required=False, + action="store_true", + help="Toggle debug mode (keep temporary directories)", ) args = parser.parse_args() @@ -725,10 +712,12 @@ def main(): dl_dir=args.output_folder ) # output folder (if None a default directory is created) + if args.debug: + stb.debug_mode = True + if args.longitudinal: stb.toggle_longitudinal_version() - if args.automri: - stb.switch_to_automri_format() + if not stb.is_correct_dcm2niix(): print( f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}" diff --git a/shanoir2bidsv1.py b/shanoir2bidsv1.py deleted file mode 100755 index 7016425..0000000 --- a/shanoir2bidsv1.py +++ /dev/null @@ -1,672 +0,0 @@ -#!/usr/bin/env python3 -DESCRIPTION = """ -shanoir2bids.py is a script that allows to download a Shanoir dataset and organise it as a BIDS data structure. - The script is made to run for every project given some information provided by the user into a ".json" - configuration file. More details regarding the configuration file in the Readme.md""" -# Script to download and BIDS-like organize data on Shanoir using "shanoir_downloader.py" developed by Arthur Masson -# @Author: Malo Gaubert , Quentin Duché -# @Date: 24 Juin 2022 -import os -import sys -import zipfile -import json -import shutil -import shanoir_downloader -from dotenv import load_dotenv -from pathlib import Path -from os.path import join as opj, splitext as ops, exists as ope, dirname as opd -from glob import glob -from time import time -import datetime -from dateutil import parser - -from heudiconv.main import workflow - - -# Load environment variables -load_dotenv(dotenv_path=opj(opd(__file__), ".env")) - - -def banner_msg(msg): - """ - Print a message framed by a banner of "*" characters - :param msg: - """ - banner = "*" * (len(msg) + 6) - print(banner + "\n* ", msg, " *\n" + banner) - - -# Keys for json configuration file -K_JSON_STUDY_NAME = "study_name" -K_JSON_L_SUBJECTS = "subjects" -K_JSON_SESSION = "session" -K_JSON_DATA_DICT = "data_to_bids" -K_JSON_FIND_AND_REPLACE = "find_and_replace_subject" -K_DCM2NIIX_PATH = "dcm2niix" -K_DCM2NIIX_OPTS = "dcm2niix_options" -K_FIND = "find" -K_REPLACE = "replace" -K_JSON_DATE_FROM = ( - "date_from" # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] -) -K_JSON_DATE_TO = ( - "date_to" # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] -) -LIST_MANDATORY_KEYS_JSON = [K_JSON_STUDY_NAME, K_JSON_L_SUBJECTS, K_JSON_DATA_DICT] -LIST_AUTHORIZED_KEYS_JSON = LIST_MANDATORY_KEYS_JSON + [ - K_DCM2NIIX_PATH, - K_DCM2NIIX_OPTS, - K_JSON_DATE_FROM, - K_JSON_DATE_TO, - K_JSON_SESSION, -] - -# Define keys for data dictionary -K_BIDS_NAME = "bidsName" -K_BIDS_DIR = "bidsDir" -K_BIDS_SES = "bidsSession" -K_DS_NAME = "datasetName" - -# Define Extensions that are dealt so far by (#todo : think of other possible extensions ?) -NIFTI = ".nii" -NIIGZ = ".nii.gz" -JSON = ".json" -BVAL = ".bval" -BVEC = ".bvec" -DCM = ".dcm" - -# Shanoir parameters -SHANOIR_FILE_TYPE_NIFTI = "nifti" -SHANOIR_FILE_TYPE_DICOM = "dicom" -DEFAULT_SHANOIR_FILE_TYPE = SHANOIR_FILE_TYPE_NIFTI - -# Define error and warning messages when call to dcm2niix is not well configured in the json file -DCM2NIIX_ERR_MSG = """ERROR !! -Conversion from DICOM to nifti can not be performed. -Please provide path to your favorite dcm2niix version in your Shanoir2BIDS .json configuration file. -Add key "{key}" with the absolute path to dcm2niix version to the following file : """ -DCM2NIIX_WARN_MSG = """WARNING. You did not provide any option to the dcm2niix call. -If you want to do so, add key "{key}" to you Shanoir2BIDS configuration file :""" - - -def check_date_format(date_to_format): - # TRUE FORMAT should be: date_format = 'Y-m-dTH:M:SZ' - try: - parser.parse(date_to_format) - # If the date validation goes wrong - except ValueError: - print( - "Incorrect data format, should be YYYY-MM-DDTHH:MM:SSZ (for example: 2020-02-19T00:00:00Z)" - ) - - -def read_json_config_file(json_file): - """ - Reads a json configuration file and checks whether mandatory keys for specifying the transformation from a - Shanoir dataset to a BIDS dataset is present. - :param json_file: str, path to a json configuration file - :return: - """ - f = open(json_file) - data = json.load(f) - # Check keys - for key in data.keys(): - if not key in LIST_AUTHORIZED_KEYS_JSON: - print('Unknown key "{}" for data dictionary'.format(key)) - for key in LIST_MANDATORY_KEYS_JSON: - if not key in data.keys(): - sys.exit('Error, missing key "{}" in data dictionary'.format(key)) - - # Sets the mandatory fields for the instance of the class - study_id = data[K_JSON_STUDY_NAME] - subjects = data[K_JSON_L_SUBJECTS] - data_dict = data[K_JSON_DATA_DICT] - - # Default non-mandatory options - list_fars = [] - dcm2niix_path = None - dcm2niix_opts = None - date_from = "*" - date_to = "*" - session_id = "*" - - if K_JSON_FIND_AND_REPLACE in data.keys(): - list_fars = data[K_JSON_FIND_AND_REPLACE] - if K_DCM2NIIX_PATH in data.keys(): - dcm2niix_path = data[K_DCM2NIIX_PATH] - if K_DCM2NIIX_OPTS in data.keys(): - dcm2niix_opts = data[K_DCM2NIIX_OPTS] - if K_JSON_DATE_FROM in data.keys(): - if data[K_JSON_DATE_FROM] == "": - data_from = "*" - else: - date_from = data[K_JSON_DATE_FROM] - check_date_format(date_from) - if K_JSON_DATE_TO in data.keys(): - if data[K_JSON_DATE_TO] == "": - data_to = "*" - else: - date_to = data[K_JSON_DATE_TO] - check_date_format(date_to) - if K_JSON_SESSION in data.keys(): - session_id = data[K_JSON_SESSION] - - # Close json file and return - f.close() - return ( - study_id, - subjects, - session_id, - data_dict, - list_fars, - dcm2niix_path, - dcm2niix_opts, - date_from, - date_to, - ) - - -def generate_heuristic_file( - shanoir2bids_dict: object, path_heuristic_file: object -) -> None: - """Generate heudiconv heuristic.py file from shanoir2bids mapping dict - Parameters - ---------- - shanoir2bids_dict : - path_heuristic_file : path of the python heuristic file (.py) - """ - heuristic = f"""from heudiconv.heuristics.reproin import create_key - -def create_bids_key(dataset): - template = create_key(subdir=dataset['bidsDir'],file_suffix=dataset['bidsName'],outtype=("dicom","nii.gz")) - return template - -def get_dataset_to_key_mapping(shanoir2bids): - dataset_to_key = dict() - for dataset in shanoir2bids: - template = create_bids_key(dataset) - dataset_to_key[dataset['datasetName']] = template - return dataset_to_key - - -def infotodict(seqinfo): - - info = dict() - shanoir2bids = {shanoir2bids_dict} - - dataset_to_key = get_dataset_to_key_mapping(shanoir2bids) - for seq in seqinfo: - if seq.series_description in dataset_to_key.keys(): - key = dataset_to_key[seq.series_description] - if key in info.keys(): - info[key].append(seq.series_id) - else: - info[key] = [seq.series_id] - return info -""" - - with open(path_heuristic_file, "w", encoding="utf-8") as file: - file.write(heuristic) - file.close() - pass - - -class DownloadShanoirDatasetToBIDS: - """ - class that handles the downloading of shanoir data set and the reformatting as a BIDS data structure - """ - - def __init__(self): - """ - Initialize the class instance - """ - self.shanoir_subjects = None # List of Shanoir subjects - self.shanoir2bids_dict = ( - None # Dictionary specifying how to reformat data into BIDS structure - ) - self.shanoir_username = None # Shanoir username - self.shanoir_study_id = None # Shanoir study ID - self.shanoir_session_id = None # Shanoir study ID - self.shanoir_file_type = ( - DEFAULT_SHANOIR_FILE_TYPE # Default download type (nifti/dicom) - ) - self.json_config_file = None - self.list_fars = [] # List of substrings to edit in subjects names - self.dl_dir = None # download directory, where data will be stored - self.parser = None # Shanoir Downloader Parser - self.n_seq = 0 # Number of sequences in the shanoir2bids_dict - self.log_fn = None - self.dcm2niix_path = None # Path to the dcm2niix the user wants to use - self.dcm2niix_opts = None # Options to add to the dcm2niix call - self.dcm2niix_config_file = None # .json file to store dcm2niix options - self.date_from = None - self.date_to = None - self.longitudinal = False - self.to_automri_format = ( - False # Special filenames for automri (close to BIDS format) - ) - self.add_sns = False # Add series number suffix to filename - - def set_json_config_file(self, json_file): - """ - Sets the configuration for the download through a json file - :param json_file: str, path to the json_file - """ - self.json_config_file = json_file - ( - study_id, - subjects, - session_id, - data_dict, - list_fars, - dcm2niix_path, - dcm2niix_opts, - date_from, - date_to, - ) = read_json_config_file(json_file=json_file) - self.set_shanoir_study_id(study_id=study_id) - self.set_shanoir_subjects(subjects=subjects) - self.set_shanoir_session_id(session_id=session_id) - self.set_shanoir2bids_dict(data_dict=data_dict) - self.set_shanoir_list_find_and_replace(list_fars=list_fars) - self.set_dcm2niix_parameters( - dcm2niix_path=dcm2niix_path, dcm2niix_opts=dcm2niix_opts - ) - self.set_date_from(date_from=date_from) - self.set_date_to(date_to=date_to) - - def set_shanoir_file_type(self, shanoir_file_type): - if shanoir_file_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI]: - self.shanoir_file_type = shanoir_file_type - else: - sys.exit("Unknown shanoir file type {}".format(shanoir_file_type)) - - def set_shanoir_study_id(self, study_id): - self.shanoir_study_id = study_id - - def set_shanoir_username(self, shanoir_username): - self.shanoir_username = shanoir_username - - def set_shanoir_domaine(self, shanoir_domaine): - self.shanoir_domaine = shanoir_domaine - - def set_shanoir_subjects(self, subjects): - self.shanoir_subjects = subjects - - def set_shanoir_session_id(self, session_id): - self.shanoir_session_id = session_id - - def set_shanoir_list_find_and_replace(self, list_fars): - self.list_fars = list_fars - - def set_dcm2niix_parameters(self, dcm2niix_path, dcm2niix_opts): - self.dcm2niix_path = dcm2niix_path - self.dcm2niix_opts = dcm2niix_opts - - def set_dcm2niix_config_files(self, path_dcm2niix_options_files): - self.dcm2niix_config_file = path_dcm2niix_options_files - # Serializing json - json_object = json.dumps(self.dcm2niix_opts, indent=4) - with open(self.dcm2niix_config_file, "w") as file: - file.write(json_object) - - def set_date_from(self, date_from): - self.date_from = date_from - - def set_date_to(self, date_to): - self.date_to = date_to - - def set_shanoir2bids_dict(self, data_dict): - self.shanoir2bids_dict = data_dict - self.n_seq = len(self.shanoir2bids_dict) - - def set_download_directory(self, dl_dir): - if dl_dir is None: - # Create a default download directory - dt = datetime.datetime.now().strftime("%Y_%m_%d_at_%Hh%Mm%Ss") - self.dl_dir = "_".join( - ["shanoir2bids", "download", self.shanoir_study_id, dt] - ) - print( - "A NEW DEFAULT directory is created as you did not provide a download directory (-of option)\n\t" - + self.dl_dir - ) - else: - self.dl_dir = dl_dir - # Create directory if it does not exist - if not ope(self.dl_dir): - Path(self.dl_dir).mkdir(parents=True, exist_ok=True) - self.set_log_filename() - - def set_heuristic_file(self, path_heuristic_file): - if path_heuristic_file is None: - print("TO BE DONE") - else: - self.heuristic_file = path_heuristic_file - - def set_log_filename(self): - curr_time = datetime.datetime.now() - basename = "shanoir_downloader_{:04}{:02}{:02}_{:02}{:02}{:02}.log".format( - curr_time.year, - curr_time.month, - curr_time.day, - curr_time.hour, - curr_time.minute, - curr_time.second, - ) - self.log_fn = opj(self.dl_dir, basename) - - def toggle_longitudinal_version(self): - self.longitudinal = True - - def switch_to_automri_format(self): - self.to_automri_format = True - - def add_series_number_suffix(self): - self.add_sns = True - - def configure_parser(self): - """ - Configure the parser and the configuration of the shanoir_downloader - """ - self.parser = shanoir_downloader.create_arg_parser() - shanoir_downloader.add_common_arguments(self.parser) - shanoir_downloader.add_configuration_arguments(self.parser) - shanoir_downloader.add_search_arguments(self.parser) - shanoir_downloader.add_ids_arguments(self.parser) - - def download_subject(self, subject_to_search): - """ - For a single subject - 1. Downloads the Shanoir datasets - 2. Reorganises the Shanoir dataset as BIDS format as defined in the json configuration file provided by user - :param subject_to_search: - :return: - """ - banner_msg("Downloading subject " + subject_to_search) - - # Open log file to write the steps of processing (downloading, renaming...) - fp = open(self.log_fn, "a") - - # Real Shanoir2Bids mapping ( deal with search terms are included in datasetName field) - bids_mapping = [] - - # Loop on each sequence defined in the dictionary - for seq in range(self.n_seq): - # Isolate elements that are called many times - shanoir_seq_name = self.shanoir2bids_dict[seq][ - K_DS_NAME - ] # Shanoir sequence name (OLD) - bids_seq_subdir = self.shanoir2bids_dict[seq][ - K_BIDS_DIR - ] # Sequence BIDS subdirectory name (NEW) - bids_seq_name = self.shanoir2bids_dict[seq][ - K_BIDS_NAME - ] # Sequence BIDS nickname (NEW) - if self.longitudinal: - bids_seq_session = self.shanoir2bids_dict[seq][ - K_BIDS_SES - ] # Sequence BIDS nickname (NEW) - - # Print message concerning the sequence that is being downloaded - print( - "\t-", - bids_seq_name, - subject_to_search, - "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", - ) - - # Initialize the parser - search_txt = ( - "studyName:" - + self.shanoir_study_id.replace(" ", "?") - + " AND datasetName:" - + shanoir_seq_name.replace(" ", "?") - + " AND subjectName:" - + subject_to_search.replace(" ", "?") - + " AND examinationComment:" - + self.shanoir_session_id.replace(" ", "*") - + " AND examinationDate:[" - + self.date_from - + " TO " - + self.date_to - + "]" - ) - - args = self.parser.parse_args( - [ - "-u", - self.shanoir_username, - "-d", - self.shanoir_domaine, - "-of", - self.dl_dir, - "-em", - "-st", - search_txt, - "-s", - "200", - "-f", - self.shanoir_file_type, - "-so", - "id,ASC", - "-t", - "500", - ] - ) # Increase time out for heavy files - - config = shanoir_downloader.initialize(args) - response = shanoir_downloader.solr_search(config, args) - - # From response, process the data - # Print the number of items found and a list of these items - if response.status_code == 200: - # Invoke shanoir_downloader to download all the data - shanoir_downloader.download_search_results(config, args, response) - - if len(response.json()["content"]) == 0: - warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns -a result on the website. -Search Text : "{}" \n""".format( - search_txt - ) - print(warn_msg) - fp.write(warn_msg) - else: - for item in response.json()["content"]: - # Define subject_id - su_id = item["subjectName"] - # If the user has defined a list of edits to subject names... then do the find and replace - for far in self.list_fars: - su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) - - # ID of the subject (sub-*) - subject_id = su_id - # correct BIDS mapping of the searched dataset - bids_seq_mapping = { - "datasetName": item["datasetName"], - "bidsDir": bids_seq_subdir, - "bidsName": bids_seq_name, - "bids_subject_id": subject_id, - } - - if self.longitudinal: - bids_seq_mapping["bids_session_id"] = bids_seq_session - else: - bids_seq_mapping["bids_session_id"] = None - - bids_mapping.append(bids_seq_mapping) - - # Write the information on the data in the log file - fp.write("- datasetId = " + str(item["datasetId"]) + "\n") - fp.write(" -- studyName: " + item["studyName"] + "\n") - fp.write(" -- subjectName: " + item["subjectName"] + "\n") - fp.write(" -- session: " + item["examinationComment"] + "\n") - fp.write(" -- datasetName: " + item["datasetName"] + "\n") - fp.write( - " -- examinationDate: " + item["examinationDate"] + "\n" - ) - fp.write(" >> Downloading archive OK\n") - - # Create temp directory to make sure the directory is empty before - # TODO: Replace with temp directory ? - tmp_dir = opj(self.dl_dir, "temp_archive") - Path(tmp_dir).mkdir(parents=True, exist_ok=True) - - # Extract the downloaded archive - dl_archive = glob(opj(self.dl_dir, "*" + item["id"] + "*.zip"))[ - 0 - ] - with zipfile.ZipFile(dl_archive, "r") as zip_ref: - extraction_dir = opj(tmp_dir, item["id"]) - zip_ref.extractall(extraction_dir) - - fp.write( - " >> Extraction of all files from archive '" - + dl_archive - + " into " - + tmp_dir - + item["id"] - + "\n" - ) - - elif response.status_code == 204: - banner_msg("ERROR : No file found!") - fp.write(" >> ERROR : No file found!\n") - else: - banner_msg( - "ERROR : Returned by the request: status of the response = " - + response.status_code - ) - fp.write( - " >> ERROR : Returned by the request: status of the response = " - + str(response.status_code) - + "\n" - ) - # Generate Heudiconv heuristic file from .json mapping - generate_heuristic_file(bids_mapping, self.heuristic_file) - # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options - - if self.longitudinal: - workflow( - files=glob(opj(self.dl_dir, 'temp_archive', "*", "*.dcm"), recursive=True), - outdir=opj(self.dl_dir, "test"), - subjs=[subject_id], - session = bids_seq_session, - converter="dcm2niix", - heuristic=self.heuristic_file, - bids_options='--bids', - dcmconfig=self.dcm2niix_config_file, - datalad=True, - minmeta=True, - ) - else: - - workflow( - files=glob(opj(self.dl_dir,'temp_archive',"*", "*.dcm"), recursive=True), - outdir=opj(self.dl_dir, "test"), - subjs=[subject_id], - converter="dcm2niix", - heuristic=self.heuristic_file, - bids_options='--bids', - dcmconfig=self.dcm2niix_config_file, - datalad=True, - minmeta=True, - ) - fp.close() - - def download(self): - """ - Loop over the Shanoir subjects and go download the required datasets - :return: - """ - self.set_log_filename() - self.configure_parser() # Configure the shanoir_downloader parser - fp = open(self.log_fn, "w") - for subject_to_search in self.shanoir_subjects: - t_start_subject = time() - self.download_subject(subject_to_search=subject_to_search) - dur_min = int((time() - t_start_subject) // 60) - dur_sec = int((time() - t_start_subject) % 60) - end_msg = ( - "Downloaded dataset for subject " - + subject_to_search - + " in {}m{}s".format(dur_min, dur_sec) - ) - banner_msg(end_msg) - - -def main(): - # Parse argument for the script - parser = shanoir_downloader.create_arg_parser(description=DESCRIPTION) - # Use username and output folder arguments from shanoir_downloader - shanoir_downloader.add_username_argument(parser) - parser.add_argument( - "-d", - "--domain", - default="shanoir.irisa.fr", - help="The shanoir domain to query.", - ) - parser.add_argument( - "-f", - "--format", - default="dicom", - choices=["dicom"], - help="The format to download.", - ) - shanoir_downloader.add_output_folder_argument(parser=parser, required=False) - # Add the argument for the configuration file - parser.add_argument( - "-j", - "--config_file", - required=True, - help="Path to the .json configuration file specifying parameters for shanoir downloading.", - ) - parser.add_argument( - "-L", - "--longitudinal", - required=False, - action="store_true", - help="Toggle longitudinal approach.", - ) - # parser.add_argument( - # "-a", "--automri", action="store_true", help="Switch to automri file tree." - # ) - # parser.add_argument( - # "-A", - # "--add_sns", - # action="store_true", - # help="Add series number suffix (compatible with -a)", - # ) - # Parse arguments - args = parser.parse_args() - - # Start configuring the DownloadShanoirDatasetToBids class instance - stb = DownloadShanoirDatasetToBIDS() - stb.set_shanoir_username(args.username) - stb.set_shanoir_domaine(args.domain) - stb.set_json_config_file( - json_file=args.config_file - ) # path to json configuration file - stb.set_shanoir_file_type(shanoir_file_type=args.format) # Format (dicom or nifti) - stb.set_download_directory( - dl_dir=args.output_folder - ) # output folder (if None a default directory is created) - stb.set_heuristic_file(path_heuristic_file="/home/alpron/heuristic.py") - stb.set_dcm2niix_config_files( - path_dcm2niix_options_files="/home/alpron/dcm2niix_options.json" - ) - if args.longitudinal: - stb.toggle_longitudinal_version() - # if args.automri: - # stb.switch_to_automri_format() - # if args.add_sns: - # if not args.automri: - # print("Warning : -A option is only compatible with -a option.") - # stb.add_series_number_suffix() - - stb.download() - - -if __name__ == "__main__": - main() From 669b85ed2606dc696b3c977fc41e435e1e107598 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 21 May 2024 10:39:00 +0200 Subject: [PATCH 81/86] [FIX]: typo --- s2b_example_config.json | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index 5407eba..d46a207 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -1,15 +1,30 @@ { - "study_name": "Aneravimm", - "subjects": ["VS_Aneravimm_010", "VS_Aneravimm_011"], - "data_to_bids": - [ - {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", "bidsName": "acq-mprage_T1w"}, - {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "acq-hr_T2w"}, - {"datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", "bidsName": "task-restingState_acq-hipp_dir-AP_bold"}, - {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b3000_dir-AP_dwi"}, - {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-PA_dwi"} - ], - "dcm2niix": "/home/alpron/softs/miniconda3/bin/dcm2niix", + "study_name": "Aneravimm", + "subjects": ["VS_Aneravimm_010", "VS_Aneravimm_011"], + "data_to_bids": [ + { + "datasetName": "t1_mprage_sag_p2_iso", + "bidsDir": "anat", + "bidsName": "t1w-mprage", + }, + {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "t2-hr"}, + { + "datasetName": "Resting State_bold AP 1.6mm HIPP", + "bidsDir": "func", + "bidsName": "ap-hipp" + }, + { + "datasetName": "Diff cusp66 b3000 AP 1.5mm", + "bidsDir": "dwi", + "bidsName": "cusp66-ap-b3000" + }, + { + "datasetName": "Diff cusp66 b0 PA 1.5mm", + "bidsDir": "dwi", + "bidsName": "cusp66-ap-b0" + } + ], + "dcm2niix": "/home/alpron/softs/miniconda3/bin/dcm2niix", "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", "dcm2niix_options": { "bids_format": true, From b27d404606e429a32daf97cf09eac395ddb2ab90 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Thu, 27 Jun 2024 16:34:28 +0200 Subject: [PATCH 82/86] [ENH]: check BIDS mapping beforehand --- shanoir2bids_heudiconv.py | 68 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 63 insertions(+), 5 deletions(-) diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py index e1d1487..b8c65a9 100755 --- a/shanoir2bids_heudiconv.py +++ b/shanoir2bids_heudiconv.py @@ -24,6 +24,9 @@ from dotenv import load_dotenv from heudiconv.main import workflow +# import loggger used in heudiconv workflow +import bids_validator + # Load environment variables load_dotenv(dotenv_path=opj(opd(__file__), ".env")) @@ -242,6 +245,7 @@ def infotodict(seqinfo): pass + class DownloadShanoirDatasetToBIDS: """ class that handles the downloading of shanoir data set and the reformatting as a BIDS data structure @@ -278,7 +282,7 @@ def __init__(self): False # Special filenames for automri (No longer used ! --> BIDS format) ) self.add_sns = False # Add series number suffix to filename - self.debug_mode = False # No debug mode by default + self.debug_mode = False # No debug mode by default def set_json_config_file(self, json_file): """ @@ -424,6 +428,57 @@ def configure_parser(self): shanoir_downloader.add_search_arguments(self.parser) shanoir_downloader.add_ids_arguments(self.parser) + def is_mapping_bids(self): + """Check BIDS compliance of filenames/path used in the configuration file""" + validator = bids_validator.BIDSValidator() + + subjects = self.shanoir_subjects + list_find_and_replace = self.list_fars + if list_find_and_replace: + # normalise subjects name + normalised_subjects = [] + for subject in subjects: + for i, far in enumerate(list_find_and_replace): + if i == 0: + normalised_subject = subject + normalised_subject = normalised_subject.replace(far["find"], far["replace"]) + normalised_subjects.append(normalised_subject) + else: + normalised_subjects = subjects + + sessions = self.shanoir_session_id + extension = '.nii.gz' + + if sessions == '*': + paths = ( + "/" + "sub-" + subject + '/' + + map["bidsDir"] + '/' + + "sub-" + subject + '_' + + map["bidsName"] + extension + + for subject in normalised_subjects + for map in self.shanoir2bids_dict + ) + else: + paths = ( + "/" + "sub-" + subject + '/' + + "ses-" + session + '/' + + map["bidsDir"] + '/' + + "sub-" + subject + '_' + "ses-" + session + '_' + + map["bidsName"] + extension + + for session in sessions + for subject in normalised_subjects + for map in self.shanoir2bids_dict + ) + + bids_errors = [p for p in paths if not validator.is_bids(p)] + + if not bids_errors: + return True, bids_errors + else: + return False, bids_errors + def download_subject(self, subject_to_search): """ For a single subject @@ -520,9 +575,7 @@ def download_subject(self, subject_to_search): # Print the number of items found and a list of these items if response.status_code == 200: # Invoke shanoir_downloader to download all the data - shanoir_downloader.download_search_results( - config, args, response - ) + shanoir_downloader.download_search_results(config, args, response) if len(response.json()["content"]) == 0: warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns @@ -723,7 +776,12 @@ def main(): f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}" ) else: - stb.download() + if stb.is_mapping_bids()[0]: + stb.download() + else: + print( + f"Provided BIDS keys {stb.is_mapping_bids()[1]} are not BIDS compliant check syntax in provided configuration file {args.config_file}" + ) if __name__ == "__main__": From 5b16489b9dcb6137bd5e8f4eccb55f76d03aa33b Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Tue, 16 Jul 2024 09:33:12 +0200 Subject: [PATCH 83/86] [ENH]: enhanced documentation for shanoir2bids --- README.md | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index eaeb567..e387459 100644 --- a/README.md +++ b/README.md @@ -31,15 +31,20 @@ conda update --all conda install -c conda-forge heudiconv git-annex=*=alldep* datalad ``` ## Usage +## How to download Shanoir datasets ? -There are three scripts to download datasets: - - `shanoir_downloader.py` simply downloads datasets from a id, a list of ids, or directly from a [shanoir search as on the shanoir solr search page](https://shanoir.irisa.fr/shanoir-ng/solr-search), - - `shanoir_downloader_check.py` is a more complete tool ; it enables to download datasets (from a csv or excel file containing a list of dataset ids, or directly from a [shanoir search as on the shanoir solr search page](https://shanoir.irisa.fr/shanoir-ng/solr-search)), verify their content, anonymize them and / or encrypt them. - - `shanoir2bids.py` uses `shanoir_downloader.py` to download Shanoir datasets and reorganises them into a BIDS data structure that is specified by the user with a `.json` configuration file. An example of configuration file is provided `s2b_example_config.json`. +There are three scripts to download datasets from a Shanoir instance: -`shanoir_downloader_check.py` creates two files in the output folder: - - `downloaded_datasets.csv` records the successfully downloaded datasets, - - `missing_datasets.csv` records the datasets which could not be downloaded. +1.`shanoir_downloader.py`: downloads datasets from a id, a list of ids, or directly from a [shanoir search as on the shanoir solr search page](https://shanoir.irisa.fr/shanoir-ng/solr-search), + +2.`shanoir_downloader_check.py`: a more complete tool ; it enables to download datasets (from a csv or excel file containing a list of dataset ids, or directly from a [shanoir search as on the shanoir solr search page](https://shanoir.irisa.fr/shanoir-ng/solr-search)), verify their content, anonymize them and / or encrypt them. + +3.`shanoir2bids.py`: Download datasets from Shanoir in DICOM format and convert them into datalad datasets in BIDS format. The conversion is parameterised by a configuration file in `.json` format. + +### `shanoir_downloader_check.py` + - `shanoir_downloader_check.py` creates two files in the output folder: + - `downloaded_datasets.csv` records the successfully downloaded datasets, + - `missing_datasets.csv` records the datasets which could not be downloaded. With those two files, `shanoir_downloader_check.py` is able to resume a download session (the downloading can be interrupted any time, the tool will not redownload datasets which have already been downloaded). @@ -53,7 +58,9 @@ See `python shanoir_downloader_check.py --help` for more information. You might want to skip the anonymization process and the encryption process with the `--skip_anonymization` and `--skip_encryption` arguments respectively (or `-sa` and `-se`). -For `shanoir2bids.py`, a configuration file must be provided to transform a Shanoir dataset into a BIDS dataset. +### `shanoir2bids.py` + +A `.json` configuration file must be provided to transform a Shanoir dataset into a BIDS dataset. ``` -----------------------------[.json configuration file information]------------------------------- This file will tell the script what Shanoir datasets should be downloaded and how the data will be organised. @@ -64,31 +71,33 @@ The dictionary in the json file must have four keys : "data_to_bids": list of dict, each dictionary specifies datasets to download and BIDS format with the following keys : -> "datasetName": str, Shanoir name for the sequence to search -> "bidsDir" : str, BIDS subdirectory sequence name (eg : "anat", "func" or "dwi", ...) - -> "bidsName" : str, BIDS sequence name (eg: "t1w-mprage", "t2-hr", "cusp66-ap-b0", ...) + -> "bidsName" : str, BIDS sequence name (eg: "t1w", "acq-b0_dir-AP", ...) ``` - -An example is provided in the file `s2b_example_config.json`. +Please refer to the [BIDS starter kit](https://bids-standard.github.io/bids-starter-kit/folders_and_files/files.html) +for exhaustive templates of filenames. A BIDS compatible example is provided in the file `s2b_example_config.json`. To download longitudinal data, a key `session` and a new entry `bidsSession` in `data_to_bids` dictionaries should be defined in the JSON configuration files. Of note, only one session can be downloaded at once. Then, the key `session` is just a string, not a list as for subjects. -### Example usage - +### Download Examples +#### Raw download To download datasets, verify the content of them, anonymize them and / or encrypt them you can use a command like: `python shanoir_downloader_check.py -u username -d shanoir.irisa.fr -ids path/to/datasets_to_download.csv -of path/to/output/folder/ -se -lf path/to/downloads.log` The `example_input_check.csv` file in this repository is an example input file (the format of the `datasets_to_download.csv` file should be the same). +#### Solr search download You can also download datasets from a [SolR search](https://shanoir.irisa.fr/shanoir-ng/solr-search) as on the website: `python shanoir_downloader.py -u amasson -d shanoir.irisa.fr -of /data/amasson/test/shanoir_test4 --search_text "FLAIR" -p 1 -s 2 ` where `--search_text` is the string you would use on [the SolR search page](https://shanoir.irisa.fr/shanoir-ng/solr-search) (for example `(subjectName:(CT* OR demo*) AND studyName:etude test) OR datasetName:*flair*`). More information on the info box of the SolR search page. -`python shanoir2bids.py -c s2b_example_config.json -d my_download_dir` will download Shanoir files identified in the configuration file and saves them as a BIDS data structure into `my_download_dir` +#### BIDS download +`python shanoir2bids.py -j s2b_example_config.json -of my_download_dir --outformat nifti` will download Shanoir datasets identified in the configuration file saves them as DICOM and convert them into a BIDS datalad dataset into `my_download_dir`. -### Search usage +## About Solr Search The `--search_text` and `--expert_mode` arguments work as on the [Shanoir search page](https://shanoir.irisa.fr/shanoir-ng/solr-search). From 3765d05052f73b1f278bcf680d263d8070df43b5 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 22 Jul 2024 10:30:08 +0200 Subject: [PATCH 84/86] [BF]: fixed typos config file after rebase --- s2b_example_config.json | 29 +++++++---------------------- 1 file changed, 7 insertions(+), 22 deletions(-) diff --git a/s2b_example_config.json b/s2b_example_config.json index d46a207..fa39e77 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -2,28 +2,13 @@ "study_name": "Aneravimm", "subjects": ["VS_Aneravimm_010", "VS_Aneravimm_011"], "data_to_bids": [ - { - "datasetName": "t1_mprage_sag_p2_iso", - "bidsDir": "anat", - "bidsName": "t1w-mprage", - }, - {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "t2-hr"}, - { - "datasetName": "Resting State_bold AP 1.6mm HIPP", - "bidsDir": "func", - "bidsName": "ap-hipp" - }, - { - "datasetName": "Diff cusp66 b3000 AP 1.5mm", - "bidsDir": "dwi", - "bidsName": "cusp66-ap-b3000" - }, - { - "datasetName": "Diff cusp66 b0 PA 1.5mm", - "bidsDir": "dwi", - "bidsName": "cusp66-ap-b0" - } - ], + {"datasetName": "t1_mprage_sag_p2_iso", "bidsDir": "anat", "bidsName": "acq-mprage_T1w"}, + {"datasetName": "t2_tse_HR_cor_MTL", "bidsDir": "anat", "bidsName": "acq-hr_T2w"}, + {"datasetName": "Resting State_bold AP 1.6mm HIPP", "bidsDir": "func", "bidsName": "task-restingState_dir-AP_bold"}, + {"datasetName": "Diff cusp66 b3000 AP 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b3000_dir-AP_dwi"}, + {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-AP_dwi"}, + {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-PA_dwi"} + ], "dcm2niix": "/home/alpron/softs/miniconda3/bin/dcm2niix", "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", "dcm2niix_options": { From b066660eb9c2a67522069f6ab77accb7930e5837 Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 22 Jul 2024 13:41:46 +0200 Subject: [PATCH 85/86] [ENH] added explicit conda command installation --- shanoir2bids_heudiconv.py | 788 -------------------------------------- 1 file changed, 788 deletions(-) delete mode 100755 shanoir2bids_heudiconv.py diff --git a/shanoir2bids_heudiconv.py b/shanoir2bids_heudiconv.py deleted file mode 100755 index b8c65a9..0000000 --- a/shanoir2bids_heudiconv.py +++ /dev/null @@ -1,788 +0,0 @@ -#!/usr/bin/env python3 -DESCRIPTION = """ -shanoir2bids.py is a script that allows to download a Shanoir dataset and organise it as a BIDS data structure. - The script is made to run for every project given some information provided by the user into a ".json" - configuration file. More details regarding the configuration file in the Readme.md""" -# Script to download and BIDS-like organize data on Shanoir using "shanoir_downloader.py" developed by Arthur Masson -# @Author: Malo Gaubert , Quentin Duché -# @Date: 24 Juin 2022 - -import os -from os.path import join as opj, splitext as ops, exists as ope, dirname as opd -from glob import glob -import sys -from pathlib import Path -from time import time -import zipfile -import datetime -import tempfile -from dateutil import parser -import json -import shutil - -import shanoir_downloader -from dotenv import load_dotenv -from heudiconv.main import workflow - -# import loggger used in heudiconv workflow -import bids_validator - - -# Load environment variables -load_dotenv(dotenv_path=opj(opd(__file__), ".env")) - - -def banner_msg(msg): - """ - Print a message framed by a banner of "*" characters - :param msg: - """ - banner = "*" * (len(msg) + 6) - print(banner + "\n* ", msg, " *\n" + banner) - - -# Keys for json configuration file -K_JSON_STUDY_NAME = "study_name" -K_JSON_L_SUBJECTS = "subjects" -K_JSON_SESSION = "session" -K_JSON_DATA_DICT = "data_to_bids" -K_JSON_FIND_AND_REPLACE = "find_and_replace_subject" -K_DCM2NIIX_PATH = "dcm2niix" -K_DCM2NIIX_OPTS = "dcm2niix_options" -K_FIND = "find" -K_REPLACE = "replace" -K_JSON_DATE_FROM = ( - "date_from" # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] -) -K_JSON_DATE_TO = ( - "date_to" # examinationDate:[2014-03-21T00:00:00Z TO 2014-03-22T00:00:00Z] -) -LIST_MANDATORY_KEYS_JSON = [K_JSON_STUDY_NAME, K_JSON_L_SUBJECTS, K_JSON_DATA_DICT] -LIST_AUTHORIZED_KEYS_JSON = LIST_MANDATORY_KEYS_JSON + [ - K_DCM2NIIX_PATH, - K_DCM2NIIX_OPTS, - K_JSON_DATE_FROM, - K_JSON_DATE_TO, - K_JSON_SESSION, -] - -# Define keys for data dictionary -K_BIDS_NAME = "bidsName" -K_BIDS_DIR = "bidsDir" -K_BIDS_SES = "bidsSession" -K_DS_NAME = "datasetName" - -# Define Extensions that are dealt so far by (#todo : think of other possible extensions ?) -NIFTI = ".nii" -NIIGZ = ".nii.gz" -JSON = ".json" -BVAL = ".bval" -BVEC = ".bvec" -DCM = ".dcm" - -# Shanoir parameters -SHANOIR_FILE_TYPE_NIFTI = "nifti" -SHANOIR_FILE_TYPE_DICOM = "dicom" -DEFAULT_SHANOIR_FILE_TYPE = SHANOIR_FILE_TYPE_NIFTI - -# Define error and warning messages when call to dcm2niix is not well configured in the json file -DCM2NIIX_ERR_MSG = """ERROR !! -Conversion from DICOM to nifti can not be performed. -Please provide path to your favorite dcm2niix version in your Shanoir2BIDS .json configuration file. -Add key "{key}" with the absolute path to dcm2niix version to the following file : """ -DCM2NIIX_WARN_MSG = """WARNING. You did not provide any option to the dcm2niix call. -If you want to do so, add key "{key}" to you Shanoir2BIDS configuration file :""" - - -def create_tmp_directory(path_temporary_directory): - tmp_dir = Path(path_temporary_directory) - if tmp_dir.exists(): - shutil.rmtree(tmp_dir) - tmp_dir.mkdir(parents=True) - pass - - -def check_date_format(date_to_format): - # TRUE FORMAT should be: date_format = 'Y-m-dTH:M:SZ' - try: - parser.parse(date_to_format) - # If the date validation goes wrong - except ValueError: - print( - "Incorrect data format, should be YYYY-MM-DDTHH:MM:SSZ (for example: 2020-02-19T00:00:00Z)" - ) - - -def read_json_config_file(json_file): - """ - Reads a json configuration file and checks whether mandatory keys for specifying the transformation from a - Shanoir dataset to a BIDS dataset is present. - :param json_file: str, path to a json configuration file - :return: - """ - f = open(json_file) - data = json.load(f) - # Check keys - for key in data.keys(): - if not key in LIST_AUTHORIZED_KEYS_JSON: - print('Unknown key "{}" for data dictionary'.format(key)) - for key in LIST_MANDATORY_KEYS_JSON: - if not key in data.keys(): - sys.exit('Error, missing key "{}" in data dictionary'.format(key)) - - # Sets the mandatory fields for the instance of the class - study_id = data[K_JSON_STUDY_NAME] - subjects = data[K_JSON_L_SUBJECTS] - data_dict = data[K_JSON_DATA_DICT] - - # Default non-mandatory options - list_fars = [] - dcm2niix_path = None - dcm2niix_opts = None - date_from = "*" - date_to = "*" - session_id = "*" - - if K_JSON_FIND_AND_REPLACE in data.keys(): - list_fars = data[K_JSON_FIND_AND_REPLACE] - if K_DCM2NIIX_PATH in data.keys(): - dcm2niix_path = data[K_DCM2NIIX_PATH] - if K_DCM2NIIX_OPTS in data.keys(): - dcm2niix_opts = data[K_DCM2NIIX_OPTS] - if K_JSON_DATE_FROM in data.keys(): - if data[K_JSON_DATE_FROM] == "": - data_from = "*" - else: - date_from = data[K_JSON_DATE_FROM] - check_date_format(date_from) - if K_JSON_DATE_TO in data.keys(): - if data[K_JSON_DATE_TO] == "": - data_to = "*" - else: - date_to = data[K_JSON_DATE_TO] - check_date_format(date_to) - if K_JSON_SESSION in data.keys(): - session_id = data[K_JSON_SESSION] - - # Close json file and return - f.close() - return ( - study_id, - subjects, - session_id, - data_dict, - list_fars, - dcm2niix_path, - dcm2niix_opts, - date_from, - date_to, - ) - - -def generate_bids_heuristic_file( - shanoir2bids_dict, - path_heuristic_file, - output_type='("dicom","nii.gz")', -) -> None: - """Generate heudiconv heuristic.py file from shanoir2bids mapping dict - Parameters - ---------- - shanoir2bids_dict : - path_heuristic_file : path of the python heuristic file (.py) - """ - if output_type == "dicom": - outtype = '("dicom",)' - elif output_type == "nifti": - outtype = '("nii.gz",)' - else: - outtype = '("dicom","nii.gz")' - - heuristic = f"""from heudiconv.heuristics.reproin import create_key - -def create_bids_key(dataset): - - template = create_key(subdir=dataset['bidsDir'],file_suffix=r"run-{{item:02d}}_" + dataset['bidsName'],outtype={outtype}) - return template - -def get_dataset_to_key_mapping(shanoir2bids): - dataset_to_key = dict() - for dataset in shanoir2bids: - template = create_bids_key(dataset) - dataset_to_key[dataset['datasetName']] = template - return dataset_to_key - -def simplify_runs(info): - info_final = dict() - for key in info.keys(): - if len(info[key])==1: - new_template = key[0].replace('run-{{item:02d}}_','') - new_key = (new_template, key[1], key[2]) - info_final[new_key] = info[key] - else: - info_final[key] = info[key] - return info_final - -def infotodict(seqinfo): - - info = dict() - shanoir2bids = {shanoir2bids_dict} - - dataset_to_key = get_dataset_to_key_mapping(shanoir2bids) - for seq in seqinfo: - if seq.series_description in dataset_to_key.keys(): - key = dataset_to_key[seq.series_description] - if key in info.keys(): - info[key].append(seq.series_id) - else: - info[key] = [seq.series_id] - # remove run- key if not needed (one run only) - info_final = simplify_runs(info) - return info_final -""" - - with open(path_heuristic_file, "w", encoding="utf-8") as file: - file.write(heuristic) - pass - - - -class DownloadShanoirDatasetToBIDS: - """ - class that handles the downloading of shanoir data set and the reformatting as a BIDS data structure - """ - - def __init__(self): - """ - Initialize the class instance - """ - self.shanoir_subjects = None # List of Shanoir subjects - self.shanoir2bids_dict = ( - None # Dictionary specifying how to reformat data into BIDS structure - ) - self.shanoir_username = None # Shanoir username - self.shanoir_study_id = None # Shanoir study ID - self.shanoir_session_id = None # Shanoir study ID - self.shanoir_file_type = SHANOIR_FILE_TYPE_DICOM # Download File Type (DICOM) - self.output_file_type = ( - DEFAULT_SHANOIR_FILE_TYPE # Default Export File Type (NIFTI) - ) - self.json_config_file = None - self.list_fars = [] # List of substrings to edit in subjects names - self.dl_dir = None # download directory, where data will be stored - self.parser = None # Shanoir Downloader Parser - self.n_seq = 0 # Number of sequences in the shanoir2bids_dict - self.log_fn = None - self.dcm2niix_path = None # Path to the dcm2niix the user wants to use - self.actual_dcm2niix_path = shutil.which("dcm2niix") - self.dcm2niix_opts = None # Options to add to the dcm2niix call - self.date_from = None - self.date_to = None - self.longitudinal = False - self.to_automri_format = ( - False # Special filenames for automri (No longer used ! --> BIDS format) - ) - self.add_sns = False # Add series number suffix to filename - self.debug_mode = False # No debug mode by default - - def set_json_config_file(self, json_file): - """ - Sets the configuration for the download through a json file - :param json_file: str, path to the json_file - """ - self.json_config_file = json_file - ( - study_id, - subjects, - session_id, - data_dict, - list_fars, - dcm2niix_path, - dcm2niix_opts, - date_from, - date_to, - ) = read_json_config_file(json_file=json_file) - self.set_shanoir_study_id(study_id=study_id) - self.set_shanoir_subjects(subjects=subjects) - self.set_shanoir_session_id(session_id=session_id) - self.set_shanoir2bids_dict(data_dict=data_dict) - self.set_shanoir_list_find_and_replace(list_fars=list_fars) - self.set_dcm2niix_parameters( - dcm2niix_path=dcm2niix_path, dcm2niix_opts=dcm2niix_opts - ) - self.set_date_from(date_from=date_from) - self.set_date_to(date_to=date_to) - - def set_output_file_type(self, outfile_type): - if outfile_type in [SHANOIR_FILE_TYPE_DICOM, SHANOIR_FILE_TYPE_NIFTI, "both"]: - self.output_file_type = outfile_type - else: - sys.exit("Unknown output file type {}".format(outfile_type)) - - def set_shanoir_study_id(self, study_id): - self.shanoir_study_id = study_id - - def set_shanoir_username(self, shanoir_username): - self.shanoir_username = shanoir_username - - def set_shanoir_domaine(self, shanoir_domaine): - self.shanoir_domaine = shanoir_domaine - - def set_shanoir_subjects(self, subjects): - self.shanoir_subjects = subjects - - def set_shanoir_session_id(self, session_id): - self.shanoir_session_id = session_id - - def set_shanoir_list_find_and_replace(self, list_fars): - self.list_fars = list_fars - - def set_dcm2niix_parameters(self, dcm2niix_path, dcm2niix_opts): - self.dcm2niix_path = dcm2niix_path - self.dcm2niix_opts = dcm2niix_opts - - def export_dcm2niix_config_options(self, path_dcm2niix_options_file): - # Serializing json - json_object = json.dumps(self.dcm2niix_opts, indent=4) - with open(path_dcm2niix_options_file, "w") as file: - file.write(json_object) - - def set_date_from(self, date_from): - self.date_from = date_from - - def set_date_to(self, date_to): - self.date_to = date_to - - def set_shanoir2bids_dict(self, data_dict): - self.shanoir2bids_dict = data_dict - self.n_seq = len(self.shanoir2bids_dict) - - def set_download_directory(self, dl_dir): - if dl_dir is None: - # Create a default download directory - dt = datetime.datetime.now().strftime("%Y_%m_%d_at_%Hh%Mm%Ss") - self.dl_dir = "_".join( - ["shanoir2bids", "download", self.shanoir_study_id, dt] - ) - print( - "A NEW DEFAULT directory is created as you did not provide a download directory (-of option)\n\t" - + self.dl_dir - ) - else: - self.dl_dir = dl_dir - # Create directory if it does not exist - if not ope(self.dl_dir): - Path(self.dl_dir).mkdir(parents=True, exist_ok=True) - self.set_log_filename() - - def set_heuristic_file(self, path_heuristic_file): - if path_heuristic_file is None: - print(f"No heuristic file provided") - else: - filename, ext = ops(path_heuristic_file) - if ext != ".py": - print( - f"Provided heuristic file {path_heuristic_file} is not a .py file as expected" - ) - else: - self.heuristic_file = path_heuristic_file - - def set_log_filename(self): - curr_time = datetime.datetime.now() - basename = "shanoir_downloader_{:04}{:02}{:02}_{:02}{:02}{:02}.log".format( - curr_time.year, - curr_time.month, - curr_time.day, - curr_time.hour, - curr_time.minute, - curr_time.second, - ) - self.log_fn = opj(self.dl_dir, basename) - - def toggle_longitudinal_version(self): - self.longitudinal = True - - def is_correct_dcm2niix(self): - current_version = Path(self.actual_dcm2niix_path) - config_version = Path(self.dcm2niix_path) - if current_version is not None and config_version is not None: - return config_version.samefile(current_version) - else: - return False - - def configure_parser(self): - """ - Configure the parser and the configuration of the shanoir_downloader - """ - self.parser = shanoir_downloader.create_arg_parser() - shanoir_downloader.add_username_argument(self.parser) - shanoir_downloader.add_domain_argument(self.parser) - self.parser.add_argument( - "-f", - "--format", - default="dicom", - choices=["dicom"], - help="The format to download.", - ) - shanoir_downloader.add_output_folder_argument(self.parser) - shanoir_downloader.add_configuration_arguments(self.parser) - shanoir_downloader.add_search_arguments(self.parser) - shanoir_downloader.add_ids_arguments(self.parser) - - def is_mapping_bids(self): - """Check BIDS compliance of filenames/path used in the configuration file""" - validator = bids_validator.BIDSValidator() - - subjects = self.shanoir_subjects - list_find_and_replace = self.list_fars - if list_find_and_replace: - # normalise subjects name - normalised_subjects = [] - for subject in subjects: - for i, far in enumerate(list_find_and_replace): - if i == 0: - normalised_subject = subject - normalised_subject = normalised_subject.replace(far["find"], far["replace"]) - normalised_subjects.append(normalised_subject) - else: - normalised_subjects = subjects - - sessions = self.shanoir_session_id - extension = '.nii.gz' - - if sessions == '*': - paths = ( - "/" + "sub-" + subject + '/' + - map["bidsDir"] + '/' + - "sub-" + subject + '_' + - map["bidsName"] + extension - - for subject in normalised_subjects - for map in self.shanoir2bids_dict - ) - else: - paths = ( - "/" + "sub-" + subject + '/' + - "ses-" + session + '/' + - map["bidsDir"] + '/' + - "sub-" + subject + '_' + "ses-" + session + '_' + - map["bidsName"] + extension - - for session in sessions - for subject in normalised_subjects - for map in self.shanoir2bids_dict - ) - - bids_errors = [p for p in paths if not validator.is_bids(p)] - - if not bids_errors: - return True, bids_errors - else: - return False, bids_errors - - def download_subject(self, subject_to_search): - """ - For a single subject - 1. Downloads the Shanoir datasets - 2. Reorganises the Shanoir dataset as BIDS format as defined in the json configuration file provided by user - :param subject_to_search: - :return: - """ - banner_msg("Downloading subject " + subject_to_search) - - # Open log file to write the steps of processing (downloading, renaming...) - fp = open(self.log_fn, "a") - - # Real Shanoir2Bids mapping (handle case when solr search term are included) - bids_mapping = [] - - # Manual temporary directories containing dowloaded DICOM.zip and extracted files - # (temporary directories that can be kept are not supported by pythn <3.1 - tmp_dicom = Path(self.dl_dir).joinpath("tmp_dicoms", subject_to_search) - tmp_archive = Path(self.dl_dir).joinpath( - "tmp_archived_dicoms", subject_to_search - ) - create_tmp_directory(tmp_archive) - create_tmp_directory(tmp_dicom) - - # Loop on each sequence defined in the dictionary - for seq in range(self.n_seq): - # Isolate elements that are called many times - shanoir_seq_name = self.shanoir2bids_dict[seq][ - K_DS_NAME - ] # Shanoir sequence name (OLD) - bids_seq_subdir = self.shanoir2bids_dict[seq][ - K_BIDS_DIR - ] # Sequence BIDS subdirectory name (NEW) - bids_seq_name = self.shanoir2bids_dict[seq][ - K_BIDS_NAME - ] # Sequence BIDS nickname (NEW) - if self.longitudinal: - bids_seq_session = self.shanoir2bids_dict[seq][ - K_BIDS_SES - ] # Sequence BIDS nickname (NEW) - - # Print message concerning the sequence that is being downloaded - print( - "\t-", - bids_seq_name, - subject_to_search, - "[" + str(seq + 1) + "/" + str(self.n_seq) + "]", - ) - - # Initialize the parser - search_txt = ( - "studyName:" - + self.shanoir_study_id.replace(" ", "?") - + " AND datasetName:" - + shanoir_seq_name.replace(" ", "?") - + " AND subjectName:" - + subject_to_search.replace(" ", "?") - + " AND examinationComment:" - + self.shanoir_session_id.replace(" ", "*") - + " AND examinationDate:[" - + self.date_from - + " TO " - + self.date_to - + "]" - ) - - args = self.parser.parse_args( - [ - "-u", - self.shanoir_username, - "-d", - self.shanoir_domaine, - "-of", - str(tmp_archive), - "-em", - "-st", - search_txt, - "-s", - "200", - "-f", - self.shanoir_file_type, - "-so", - "id,ASC", - "-t", - "500", - ] - ) # Increase time out for heavy files - - config = shanoir_downloader.initialize(args) - response = shanoir_downloader.solr_search(config, args) - - # From response, process the data - # Print the number of items found and a list of these items - if response.status_code == 200: - # Invoke shanoir_downloader to download all the data - shanoir_downloader.download_search_results(config, args, response) - - if len(response.json()["content"]) == 0: - warn_msg = """WARNING ! The Shanoir request returned 0 result. Make sure the following search text returns -a result on the website. -Search Text : "{}" \n""".format( - search_txt - ) - print(warn_msg) - fp.write(warn_msg) - else: - for item in response.json()["content"]: - # Define subject_id - su_id = item["subjectName"] - # If the user has defined a list of edits to subject names... then do the find and replace - for far in self.list_fars: - su_id = su_id.replace(far[K_FIND], far[K_REPLACE]) - # ID of the subject (sub-*) - subject_id = su_id - - # correct BIDS mapping of the searched dataset - bids_seq_mapping = { - "datasetName": item["datasetName"], - "bidsDir": bids_seq_subdir, - "bidsName": bids_seq_name, - "bids_subject_id": subject_id, - } - - if self.longitudinal: - bids_seq_mapping["bids_session_id"] = bids_seq_session - else: - bids_seq_session = None - - bids_seq_mapping["bids_session_id"] = bids_seq_session - - bids_mapping.append(bids_seq_mapping) - - # Write the information on the data in the log file - fp.write("- datasetId = " + str(item["datasetId"]) + "\n") - fp.write(" -- studyName: " + item["studyName"] + "\n") - fp.write(" -- subjectName: " + item["subjectName"] + "\n") - fp.write(" -- session: " + item["examinationComment"] + "\n") - fp.write(" -- datasetName: " + item["datasetName"] + "\n") - fp.write( - " -- examinationDate: " + item["examinationDate"] + "\n" - ) - fp.write(" >> Downloading archive OK\n") - - # Extract the downloaded archive - dl_archive = glob(opj(tmp_archive, "*" + item["id"] + "*.zip"))[ - 0 - ] - with zipfile.ZipFile(dl_archive, "r") as zip_ref: - extraction_dir = opj(tmp_dicom, item["id"]) - zip_ref.extractall(extraction_dir) - - fp.write( - " >> Extraction of all files from archive '" - + dl_archive - + " into " - + extraction_dir - + "\n" - ) - - elif response.status_code == 204: - banner_msg("ERROR : No file found!") - fp.write(" >> ERROR : No file found!\n") - else: - banner_msg( - "ERROR : Returned by the request: status of the response = " - + response.status_code - ) - fp.write( - " >> ERROR : Returned by the request: status of the response = " - + str(response.status_code) - + "\n" - ) - - # Launch DICOM to BIDS conversion using heudiconv + heuristic file + dcm2niix options - with tempfile.NamedTemporaryFile( - mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".py" - ) as heuristic_file: - # Generate Heudiconv heuristic file from configuration.json mapping - generate_bids_heuristic_file( - bids_mapping, heuristic_file.name, output_type=self.output_file_type - ) - with tempfile.NamedTemporaryFile( - mode="r+", encoding="utf-8", dir=self.dl_dir, suffix=".json" - ) as dcm2niix_config_file: - self.export_dcm2niix_config_options(dcm2niix_config_file.name) - workflow_params = { - "files": glob(opj(tmp_dicom, "*", "*.dcm"), recursive=True), - "outdir": opj(self.dl_dir, str(self.shanoir_study_id)), - "subjs": [subject_id], - "converter": "dcm2niix", - "heuristic": heuristic_file.name, - "bids_options": "--bids", - # "with_prov": True, - "dcmconfig": dcm2niix_config_file.name, - "datalad": True, - "minmeta": True, - "grouping": "all", # other options are too restrictive (tested on EMISEP) - } - - if self.longitudinal: - workflow_params["session"] = bids_seq_session - - workflow(**workflow_params) - fp.close() - if not self.debug_mode: - shutil.rmtree(tmp_archive.parent) - shutil.rmtree(tmp_dicom.parent) - - - def download(self): - """ - Loop over the Shanoir subjects and go download the required datasets - :return: - """ - self.set_log_filename() - self.configure_parser() # Configure the shanoir_downloader parser - fp = open(self.log_fn, "w") - for subject_to_search in self.shanoir_subjects: - t_start_subject = time() - self.download_subject(subject_to_search=subject_to_search) - dur_min = int((time() - t_start_subject) // 60) - dur_sec = int((time() - t_start_subject) % 60) - end_msg = ( - "Downloaded dataset for subject " - + subject_to_search - + " in {}m{}s".format(dur_min, dur_sec) - ) - banner_msg(end_msg) - - -def main(): - # Parse argument for the script - parser = shanoir_downloader.create_arg_parser(description=DESCRIPTION) - # Use username and output folder arguments from shanoir_downloader - shanoir_downloader.add_username_argument(parser) - parser.add_argument( - "-d", - "--domain", - default="shanoir.irisa.fr", - help="The shanoir domain to query.", - ) - - parser.add_argument( - "--outformat", - default="both", - choices=["nifti", "dicom", "both"], - help="The format to download.", - ) - - shanoir_downloader.add_output_folder_argument(parser=parser, required=False) - # Add the argument for the configuration file - parser.add_argument( - "-j", - "--config_file", - required=True, - help="Path to the .json configuration file specifying parameters for shanoir downloading.", - ) - parser.add_argument( - "-L", - "--longitudinal", - required=False, - action="store_true", - help="Toggle longitudinal approach.", - ) - parser.add_argument( - "--debug", - required=False, - action="store_true", - help="Toggle debug mode (keep temporary directories)", - ) - - args = parser.parse_args() - - # Start configuring the DownloadShanoirDatasetToBids class instance - stb = DownloadShanoirDatasetToBIDS() - stb.set_shanoir_username(args.username) - stb.set_shanoir_domaine(args.domain) - stb.set_json_config_file( - json_file=args.config_file - ) # path to json configuration file - stb.set_output_file_type(args.outformat) - stb.set_download_directory( - dl_dir=args.output_folder - ) # output folder (if None a default directory is created) - - if args.debug: - stb.debug_mode = True - - if args.longitudinal: - stb.toggle_longitudinal_version() - - if not stb.is_correct_dcm2niix(): - print( - f"Current dcm2niix path {stb.actual_dcm2niix_path} is different from dcm2niix configured path {stb.dcm2niix_path}" - ) - else: - if stb.is_mapping_bids()[0]: - stb.download() - else: - print( - f"Provided BIDS keys {stb.is_mapping_bids()[1]} are not BIDS compliant check syntax in provided configuration file {args.config_file}" - ) - - -if __name__ == "__main__": - main() From ac64c3ecc7a942f42f3b46b0c4a7b0369f9adaae Mon Sep 17 00:00:00 2001 From: Alexandre Pron Date: Mon, 22 Jul 2024 14:27:06 +0200 Subject: [PATCH 86/86] [BF]: added bids-validator package --- pyproject.toml | 6 +++++- s2b_example_config.json | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 9ce4885..fde5251 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,11 @@ readme = "README.md" license = {file = "LICENSE"} keywords = ["Shanoir", "DICOM", "NIFTI", "BIDS"] classifiers = [ - "Programming Language :: Python" + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10" ] [project.urls] diff --git a/s2b_example_config.json b/s2b_example_config.json index fa39e77..587c20f 100644 --- a/s2b_example_config.json +++ b/s2b_example_config.json @@ -9,7 +9,7 @@ {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-AP_dwi"}, {"datasetName": "Diff cusp66 b0 PA 1.5mm", "bidsDir": "dwi", "bidsName": "acq-b0_dir-PA_dwi"} ], - "dcm2niix": "/home/alpron/softs/miniconda3/bin/dcm2niix", + "dcm2niix": "/home/alpron/softs/miniconda3/envs/test-env/bin/dcm2niix", "dcm2niix_options_comment": "dcm2niix configuration options in the nipype format (see https://nipype.readthedocs.io/en/latest/api/generated/nipype.interfaces.dcm2nii.html)", "dcm2niix_options": { "bids_format": true,